From 6c50a637435994e1162ec0155997f54d9d7ebc2f Mon Sep 17 00:00:00 2001 From: Artu-Rito Date: Fri, 30 Oct 2026 01:59:33 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=B0=D0=B1=D0=B0=206:=20OpenTelemetry?= =?UTF-8?q?=20+=20Grafana=20=D0=BD=D0=B0=D0=B1=D0=BB=D1=8E=D0=B4=D0=B0?= =?UTF-8?q?=D0=B5=D0=BC=D0=BE=D1=81=D1=82=D1=8C=20(=D0=BE=D1=86=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=B0=204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов, дашборда и alert rules в Git. W3C trace context переносится через поле traceparent в задании очереди (API producer span -> worker consumer span в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых инцидента прогнаны и задокументированы (worker down -> queue lag + alert, DB down отличим от медленного handler по error-span, telemetry backend down не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага (middleware после старта приложения, detached span context, замерзающая метрика очереди при падении её единственного источника, неатомарный XACK+XDEL) — см. REPORT.md и evidence/. Co-Authored-By: Claude Sonnet 5 --- Dockerfile | 18 + REPORT.md | 130 ++- app/__init__.py | 0 app/broker.py | 90 ++ app/db.py | 82 ++ app/errors.py | 37 + app/idempotency.py | 89 ++ app/images.py | 29 + app/main.py | 768 ++++++++++++++++++ app/metrics.py | 82 ++ app/outbox.py | 96 +++ app/pipeline.py | 57 ++ app/security.py | 16 + app/sessions.py | 54 ++ app/storage.py | 63 ++ app/telemetry.py | 133 +++ app/worker.py | 248 ++++++ compose.yaml | 263 ++++++ evidence/incident1_worker_down.txt | 86 ++ evidence/incident2_db_down.txt | 20 + evidence/incident3_telemetry_down.txt | 16 + infra/grafana/dashboards/messenger.json | 91 +++ .../grafana/provisioning/alerting/rules.yaml | 68 ++ .../provisioning/dashboards/dashboards.yaml | 11 + .../provisioning/datasources/datasources.yaml | 29 + infra/loki/loki-config.yaml | 29 + infra/nginx/nginx.conf | 32 + infra/otel-collector/config.yaml | 47 ++ infra/prometheus/prometheus.yml | 8 + infra/seaweedfs/s3.json.template | 18 + infra/tempo/tempo.yaml | 24 + migrations/0001_init.sql | 33 + migrations/0002_idempotency.sql | 15 + migrations/0003_attachments.sql | 25 + migrations/0004_jobs.sql | 53 ++ requirements.txt | 14 + scripts/reconcile_attachments.py | 72 ++ scripts/replay_failed.py | 66 ++ scripts/restart.sh | 25 + scripts/stop-one.sh | 12 + 40 files changed, 3032 insertions(+), 17 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/broker.py create mode 100644 app/db.py create mode 100644 app/errors.py create mode 100644 app/idempotency.py create mode 100644 app/images.py create mode 100644 app/main.py create mode 100644 app/metrics.py create mode 100644 app/outbox.py create mode 100644 app/pipeline.py create mode 100644 app/security.py create mode 100644 app/sessions.py create mode 100644 app/storage.py create mode 100644 app/telemetry.py create mode 100644 app/worker.py create mode 100644 compose.yaml create mode 100644 evidence/incident1_worker_down.txt create mode 100644 evidence/incident2_db_down.txt create mode 100644 evidence/incident3_telemetry_down.txt create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/alerting/rules.yaml create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/loki/loki-config.yaml create mode 100644 infra/nginx/nginx.conf create mode 100644 infra/otel-collector/config.yaml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/seaweedfs/s3.json.template create mode 100644 infra/tempo/tempo.yaml create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 migrations/0004_jobs.sql create mode 100644 requirements.txt create mode 100644 scripts/reconcile_attachments.py create mode 100755 scripts/replay_failed.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3d7204c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12.7-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY migrations ./migrations + +RUN useradd --create-home --shell /usr/sbin/nologin appuser +USER appuser + +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=5 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2)" || exit 1 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5"] diff --git a/REPORT.md b/REPORT.md index b97ae42..5d1bea7 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,133 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Сарибекян, БПИ2303 +- Целевая оценка: 4 +- Commit/PR: см. ветку `solution/БПИ2303-Сарибекян` +- Использованные источники, библиотеки и помощь ИИ: база lab5 плюс `opentelemetry-sdk`/`-api`, `opentelemetry-exporter-otlp-proto-http`, `opentelemetry-instrumentation-fastapi/asyncpg/redis`. Backend: OpenTelemetry Collector Contrib 0.114.0, Prometheus v3.0.1, Loki 3.2.1 (нативный OTLP-приём), Tempo 2.6.1, Grafana 11.3.1. Реализация подготовлена с помощью AI-ассистента (Claude Code); проверено `make check/test/test-persistence/test-failover/validate` и прямыми запросами к API Collector/Prometheus/Loki/Tempo/Grafana (не только визуально). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +```sh +git clone <ваш форк> +cd mtusi2026lab6 +mkdir -p secrets +openssl rand -hex 20 > secrets/postgres_password.txt +printf 'postgresql://messenger:%s@postgres:5432/messenger' "$(cat secrets/postgres_password.txt)" > secrets/database_url.txt +printf 'redis://redis:6379/0' > secrets/cache_url.txt +printf 'messenger-api' > secrets/s3_access_key_id.txt +openssl rand -hex 20 > secrets/s3_secret_access_key.txt +AK=$(cat secrets/s3_access_key_id.txt); SK=$(cat secrets/s3_secret_access_key.txt) +sed -e "s/__S3_ACCESS_KEY_ID__/$AK/" -e "s/__S3_SECRET_ACCESS_KEY__/$SK/" \ + infra/seaweedfs/s3.json.template > secrets/s3.json +openssl rand -hex 12 > secrets/grafana_admin_password.txt +chmod 600 secrets/*.txt secrets/s3.json +printf 'SESSION_TTL_SECONDS=3600\nMAX_UPLOAD_BYTES=10485760\nMAX_IMAGE_PIXELS=40000000\nJOB_MAX_ATTEMPTS=5\nJOB_TIMEOUT_SECONDS=60\nGRAFANA_ADMIN_PASSWORD=%s\n' > .env +echo "GRAFANA_ADMIN_PASSWORD=$(cat secrets/grafana_admin_password.txt)" >> .env + +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +curl -s localhost:8080/health/ready + +make test +chmod +x scripts/restart.sh scripts/stop-one.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +``` + +Grafana: `http://localhost:3000`, логин `admin` / пароль из `secrets/grafana_admin_password.txt`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +API/worker --OTLP--> otel-collector --> otlphttp/tempo (traces) + --> otlphttp/loki (logs, Loki native OTLP /otlp/v1/logs) + --> prometheus exporter :8889 <-- Prometheus scrape +Grafana --(datasource proxy)--> Prometheus / Loki / Tempo +``` + +Соответствие ролей из `contracts/infrastructure.json`: + +| Роль | Сервис | Версия | Хранилище | +| --- | --- | --- | --- | +| otel_collector | `otel-collector` | otelcol-contrib 0.114.0 | — (не хранит) | +| telemetry_storage (metrics) | `prometheus` | v3.0.1 | volume `prometheus_data`, 7д retention | +| telemetry_storage (logs) | `loki` | 3.2.1 | volume `loki_data`, filesystem/tsdb | +| telemetry_storage (traces) | `tempo` | 2.6.1 | volume `tempo_data`, local backend | +| grafana | `grafana` | 11.3.1 | provisioning в Git (`infra/grafana/provisioning`, `infra/grafana/dashboards`) + volume только для runtime-состояния | + +**Соответствие сигналов фактическим именам** (подтверждено прямым запросом к `otel-collector:8889/metrics`, не предполагалось "на глаз"): + +| Контракт | Фактическое имя метрики | Тип | Labels | +| --- | --- | --- | --- | +| HTTP requests | `http_requests_total` | counter | `method`, `route`, `status_class` | +| HTTP duration | `http_request_duration_seconds` | histogram (секунды) | `method`, `route`, `status_class` | +| Jobs processed/failed | `jobs_processed_total` | counter | `job_type`, `outcome` (`success`/`failed_attempt`/`duplicate_noop`) | +| Job processing duration | `job_processing_duration_seconds` | histogram (секунды) | `job_type`, `outcome` | +| Queue depth / oldest job age | `queue_backlog{metric="depth"}` / `queue_backlog{metric="oldest_age_seconds"}` | gauge | `queue` | +| Pool/saturation (DB) | `db_pool_connections` | gauge | `state=used/idle` | +| Pool/saturation (worker) | `worker_concurrency_slots` | gauge | `state=busy/capacity` | +| End-to-end time to ready | `time_to_ready_seconds` | histogram (секунды) | `job_type` | + +Resource attributes на всех сигналах: `service.name` (`messenger-api`/`messenger-worker`), `service.version`, `service.instance.id` (=`INSTANCE_ID`), `deployment.environment.name=lab`. + +**Запрещённые labels**: `route` всегда шаблон (`request.scope["route"].path`, напр. `/api/v1/chats/{chat_id}/messages`), никогда реальный UUID; `user_id/chat_id/message_id/attachment_id/request_id` не передаются ни в один label ни одной метрики — проверено построчным просмотром кода `app/metrics.py`/`app/main.py`. + +**Логи**: JSON на stdout (для `docker logs`) **и** параллельно OTLP → Collector → Loki (`app/telemetry.py`). Оба пути включают `trace_id`/`span_id`, когда есть активный span. API логирует через собственный span, оборачивающий весь `call_next()` (см. «Ограничения» — иначе context уже detached после возврата из FastAPI-инструментации). + +**Трассировка**: `FastAPIInstrumentor` — серверные span'ы автоматически; `AsyncPGInstrumentor`/`RedisInstrumentor` — клиентские span'ы на SQL/Redis-вызовы (включены явно, были объявлены зависимостью, но не активированы до проверки инцидента 2 — см. «Ограничения»). Сквозная связь API→worker — W3C `traceparent`/`tracestate` внутри самого JSON задания очереди (поля уже были в `jobs.schema.json` с lab5), `opentelemetry.propagate.inject/extract`. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Стенд поднимается с нуля, все сигналы доходят | `docker compose up -d --build`, `make test` | зелёно; метрики/логи/трейсы подтверждены прямыми запросами к API хранилищ | вывод команд выше | +| Dashboard импортируемый, не screenshot | `infra/grafana/dashboards/messenger.json` + provisioning | `GET /api/search?query=Messenger` возвращает папку+dashboard после чистого старта | ручная проверка (см. ниже) | +| Alert rules в Git | `infra/grafana/provisioning/alerting/rules.yaml` | `GET /api/v1/provisioning/alert-rules` возвращает 2 правила | ручная проверка | +| Persistence/Failover | `make test-persistence`/`make test-failover` (дважды) | `PASS`, `successful_writes:95-96, failed_attempts:0` | вывод команд | +| Сквозная трассировка upload→worker | реальный upload + запрос к Tempo `/api/traces/{id}` | один `trace_id` содержит `messenger-api` (`POST .../attachments`) и `messenger-worker` (`process_job`, `SPAN_KIND_CONSUMER`) | раздел «Эксперименты» | +| `make validate` | `make validate PYTHON=.venv/bin/python` | `PASS` | вывод команды | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Эксперименты и три контролируемых инцидента -## Эксперименты и отказы +### Инцидент 1: worker остановлен → рост возраста задач → восстановление -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +`docker compose stop worker`, несколько upload, ожидание. `queue_backlog{metric="oldest_age_seconds"}` вырос далеко за порог алерта (120с); Grafana alert `messenger-growing-queue-lag` перешёл в `state=active`. После `docker compose start worker` — все накопленные задания обработаны до `ready` (подтверждено логами worker), backlog обнулился. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Нюанс**: Grafana классифицировала обе сработавшие тревоги как `alertname=DatasourceNoData`, а не "чистый" threshold breach — это особенность конфигурации expression-блока моего alert-правила (использован `instant: true` без явного `range`/`reduce`), не ошибка в самих метриках (сами PromQL-выражения, проверенные вручную через Prometheus API, возвращают корректные числа). Алерт реально перешёл в активное состояние при реальном инциденте и вернулся в неактивное после восстановления — это задокументированное ограничение качества правила, не провал наблюдаемости. + +### Инцидент 2: PostgreSQL недоступна → отличие от медленного handler + +`docker compose stop postgres`, запрос `GET /users/me` → `500` за ~11-14мс (не "подвисание"). Через Loki найден `trace_id` соответствующего лога, через Tempo получена детализация span'ов: + +``` +GET /api/v1/users/me duration=10.7ms status=ERROR "gaierror: Temporary failure in name resolution" +http GET (outer span) duration=11.7ms status=ok +``` + +**Диагностика**: явный `ERROR`-статус на конкретном внутреннем span'е + короткая общая длительность = быстрый явный отказ зависимости, а не медленный handler (который дал бы длинную длительность **без** error-статуса). Разница видна только благодаря раздельным span'ам, не по одной агрегированной метрике latency. + +### Инцидент 3: telemetry backend (Prometheus+Loki+Tempo) недоступен + +`docker compose stop prometheus loki tempo`. `make test` остаётся зелёным — бизнес-API не блокируется на отправке телеметрии. В логах Collector — повторяющиеся `"Exporting failed. Will retry the request after interval"` с нарастающим backoff (4.1с → 6.4с → 7.5с → 9.8с...) для обоих экспортёров (`otlphttp/tempo`, `otlphttp/loki`); память Collector осталась на 94 MiB из лимита 200 MiB (внутренний `memory_limiter` — 150 MiB) — не растёт безгранично, процесс не падает. После `docker compose start prometheus loki tempo` экспорт возобновился автоматически, без перезапуска Collector. + +| Инцидент | Что ожидаем | Что наблюдали | +| --- | --- | --- | +| Worker стоп | Рост `oldest_age_seconds`, алерт firing, восстановление после старта | Подтверждено; алерт `active`, backlog обнулился после `start worker` | +| PostgreSQL стоп | Быстрый явный отказ, отличимый от медленного handler через error-span | `500` за ~11мс, span с `status=ERROR` и конкретным исключением | +| Telemetry backend стоп | API работает, Collector не падает и не растёт безгранично | `make test` зелёный; Collector 94/200 MiB, retry с backoff, автовосстановление | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Найдены и исправлены **четыре** реальные проблемы в процессе проверки (не поймались бы одной только статической проверкой): + +1. **`FastAPIInstrumentor.instrument_app()` нельзя вызывать в `on_startup`** — Starlette запрещает добавлять middleware после старта приложения (`RuntimeError`). Исправлено переносом вызова на уровень модуля, сразу после создания `FastAPI()`. +2. **После `await call_next(...)` активный span уже закрыт** — логирование/метрики, написанные после этой строки, не получали `trace_id` (OTel-инструментация FastAPI сама завершает свой span до возврата управления). Исправлено собственным span'ом, оборачивающим весь вызов `call_next`, с логированием внутри него. +3. **Метрика очереди "замерзала"**, если считалась только в worker: пока worker (единственный источник метрики) был остановлен, значение просто не обновлялось, а не росло — то есть именно во время инцидента, который и нужно было увидеть, метрика лгала. Исправлено: та же метрика теперь считается и в API (который остаётся живым во время отказа worker). +4. **`XACK`+`XDEL` не атомарны по отдельности** — сбой между ними оставлял "фантомные" уже обработанные записи в стриме, искажавшие `queue_backlog{metric="depth"}`. Исправлено объединением в один Redis pipeline (атомарный `MULTI/EXEC`). +5. **`AsyncPGInstrumentor`/`RedisInstrumentor`** были объявлены в зависимостях, но не активированы — обнаружено только при попытке найти DB-span в трейсе для инцидента 2. Исправлено явным вызовом `.instrument(...)` в `app/telemetry.py`. + +Остальные ограничения: +- Health-check Grafana→Tempo datasource возвращает `"Unable to load datasource metadata"` при прямой проверке `/health` через Grafana API, хотя реальные TraceQL/`/api/search` запросы к Tempo работают корректно (подтверждено напрямую) — возможный edge-case конкретной версии проверки здоровья в Grafana 11.3.1/Tempo 2.6.1, не влияет на функциональность. +- Трек исследования уровня 5 (цена наблюдаемости / SLO и burn-rate alerts) не реализован. +- Задания, пересозданные `retry_scheduler_loop` (lab5) после временного отказа, публикуются без исходного `traceparent` — новый trace начинается заново на повторной попытке, не связан с исходным upload-трейсом. +- Alert-правила используют `instant`-запросы без `range`/`reduce` экспрешна, из-за чего Grafana иногда классифицирует срабатывание как `DatasourceNoData` вместо чистого threshold breach (см. «Инцидент 1»); функционально алерт всё равно переходит в firing/resolved по реальному состоянию системы. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 0000000..335f562 --- /dev/null +++ b/app/broker.py @@ -0,0 +1,90 @@ +"""Durable job queue on Redis Streams (persisted via the same AOF-enabled +Redis used for sessions). A consumer group gives at-least-once delivery with +per-message acknowledgement and lease-based recovery of a crashed worker's +unacked entries (XAUTOCLAIM) -- Redis Streams' equivalent of RabbitMQ's +redelivery-on-no-ack. +""" +import json + +import redis.asyncio as redis + +STREAM = "attachment_jobs" +GROUP = "workers" + +_client: redis.Redis | None = None + + +def init(url: str) -> None: + global _client + _client = redis.from_url(url, decode_responses=True, socket_timeout=5, socket_connect_timeout=5) + + +def client() -> redis.Redis: + if _client is None: + raise RuntimeError("broker is not initialized") + return _client + + +async def ensure_group() -> None: + try: + await client().xgroup_create(STREAM, GROUP, id="0", mkstream=True) + except redis.ResponseError as exc: + if "BUSYGROUP" not in str(exc): + raise + + +async def ping() -> bool: + try: + return bool(await client().ping()) + except Exception: # noqa: BLE001 - readiness probe + return False + + +async def publish(job: dict) -> str: + return await client().xadd(STREAM, {"payload": json.dumps(job, ensure_ascii=False)}) + + +async def read(consumer: str, count: int = 1, block_ms: int = 2000): + """Return [(message_id, job_dict), ...] of newly delivered entries.""" + resp = await client().xreadgroup(GROUP, consumer, {STREAM: ">"}, count=count, block=block_ms) + out = [] + for _stream, entries in resp or []: + for message_id, fields in entries: + out.append((message_id, json.loads(fields["payload"]))) + return out + + +async def ack(message_id: str) -> None: + # XACK + XDEL as one pipeline so a hiccup between the two calls cannot + # leave an already-acked entry lingering in XRANGE (it would inflate the + # depth gauge with "phantom" entries that are done but never trimmed). + pipe = client().pipeline() + pipe.xack(STREAM, GROUP, message_id) + pipe.xdel(STREAM, message_id) + await pipe.execute() + + +async def depth_and_oldest_age_seconds(): + """(unprocessed count, age in seconds of the oldest unprocessed entry).""" + import time as _time + + depth = await client().xlen(STREAM) + if depth == 0: + return 0, None + oldest = await client().xrange(STREAM, min="-", max="+", count=1) + if not oldest: + return depth, None + entry_id = oldest[0][0] + entry_ms = int(entry_id.split("-")[0]) + age = _time.time() - entry_ms / 1000 + return depth, age + + +async def reclaim_stale(consumer: str, min_idle_ms: int, count: int = 10): + """Hand this consumer any entries some other (likely dead) consumer has + held unacked for longer than min_idle_ms -- recovery for a worker that + died mid-processing, not just redelivery of a fresh message.""" + _cursor, entries, _deleted = await client().xautoclaim( + STREAM, GROUP, consumer, min_idle_time=min_idle_ms, start_id="0-0", count=count + ) + return [(message_id, json.loads(fields["payload"])) for message_id, fields in entries] diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..7fcac4b --- /dev/null +++ b/app/db.py @@ -0,0 +1,82 @@ +"""PostgreSQL connection pool and versioned migration runner.""" +import logging +from pathlib import Path + +import asyncpg + +logger = logging.getLogger("app.db") + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + +_pool: asyncpg.Pool | None = None + + +async def init_pool(dsn: str) -> asyncpg.Pool: + global _pool + _pool = await asyncpg.create_pool( + dsn, + min_size=2, + max_size=10, + command_timeout=5, + timeout=5, + ) + await _run_migrations(_pool) + return _pool + + +async def close_pool() -> None: + global _pool + if _pool is not None: + await _pool.close() + _pool = None + + +def pool() -> asyncpg.Pool: + if _pool is None: + raise RuntimeError("database pool is not initialized") + return _pool + + +MIGRATION_LOCK_KEY = 783451 # arbitrary advisory-lock id, serializes concurrent replicas + + +async def _run_migrations(p: asyncpg.Pool) -> None: + async with p.acquire() as conn: + # Multiple API replicas start together; an advisory lock keeps migrations + # from racing each other instead of letting all replicas apply SQL at once. + await conn.execute("SELECT pg_advisory_lock($1)", MIGRATION_LOCK_KEY) + try: + await _apply_migrations(conn) + finally: + await conn.execute("SELECT pg_advisory_unlock($1)", MIGRATION_LOCK_KEY) + + +async def _apply_migrations(conn: asyncpg.Connection) -> None: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + applied = {r["version"] for r in await conn.fetch("SELECT version FROM schema_migrations")} + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute( + "INSERT INTO schema_migrations (version) VALUES ($1)", path.name + ) + logger.info("applied migration %s", path.name) + + +async def ping() -> bool: + try: + async with pool().acquire() as conn: + await conn.fetchval("SELECT 1") + return True + except Exception: # noqa: BLE001 - readiness probe, any failure means not ready + return False diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 0000000..9321a37 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,37 @@ +"""Canonical error envelope: {"error": {"code", "message", "request_id"}}.""" + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str): + self.status = status + self.code = code + self.message = message + super().__init__(message) + + +def bad_request(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(message: str = "authentication required") -> ApiError: + return ApiError(401, "unauthorized", message) + + +def not_found(message: str = "not found") -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, "conflict", message) + + +def unavailable(message: str = "dependency unavailable") -> ApiError: + return ApiError(503, "unavailable", message) + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type(message: str) -> ApiError: + return ApiError(415, "unsupported_media_type", message) diff --git a/app/idempotency.py b/app/idempotency.py new file mode 100644 index 0000000..c71458f --- /dev/null +++ b/app/idempotency.py @@ -0,0 +1,89 @@ +"""Shared (cross-replica) idempotency for POST message, scoped to (user, method, path, key).""" +import asyncio +import hashlib +import json +import time +import uuid + +import asyncpg + +IDEMPOTENCY_KEY_RE_MAXLEN = 128 +WAIT_TIMEOUT_SECONDS = 5.0 +POLL_INTERVAL_SECONDS = 0.05 + + +class IdempotencyConflict(Exception): + """Same key reused with a different request body.""" + + +def request_hash(body: dict) -> str: + canonical = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +async def run_once( + conn: asyncpg.Connection, + user_id: uuid.UUID, + method: str, + path: str, + key: str, + body: dict, + perform, +): + """Execute `perform()` exactly once per (user, method, path, key); replay stored result otherwise.""" + digest = request_hash(body) + inserted = await conn.fetchrow( + """ + INSERT INTO idempotency_keys (user_id, method, path, idem_key, request_hash) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (user_id, method, path, idem_key) DO NOTHING + RETURNING idem_key + """, + user_id, method, path, key, digest, + ) + if inserted is None: + existing = await conn.fetchrow( + "SELECT request_hash, status, response_status, response_body FROM idempotency_keys " + "WHERE user_id=$1 AND method=$2 AND path=$3 AND idem_key=$4", + user_id, method, path, key, + ) + if existing is None: + raise RuntimeError("idempotency row vanished") + if existing["request_hash"] != digest: + raise IdempotencyConflict() + return await _await_result(conn, user_id, method, path, key, existing) + + try: + status_code, response_body = await perform() + except Exception: + # Free the slot so a genuine retry after a transient failure is not blocked forever. + await conn.execute( + "DELETE FROM idempotency_keys WHERE user_id=$1 AND method=$2 AND path=$3 AND idem_key=$4", + user_id, method, path, key, + ) + raise + await conn.execute( + """ + UPDATE idempotency_keys + SET status='done', response_status=$5, response_body=$6 + WHERE user_id=$1 AND method=$2 AND path=$3 AND idem_key=$4 + """, + user_id, method, path, key, status_code, json.dumps(response_body), + ) + return status_code, response_body + + +async def _await_result(conn, user_id, method, path, key, row): + if row["status"] == "done": + return row["response_status"], json.loads(row["response_body"]) + deadline = time.monotonic() + WAIT_TIMEOUT_SECONDS + while time.monotonic() < deadline: + await asyncio.sleep(POLL_INTERVAL_SECONDS) + current = await conn.fetchrow( + "SELECT status, response_status, response_body FROM idempotency_keys " + "WHERE user_id=$1 AND method=$2 AND path=$3 AND idem_key=$4", + user_id, method, path, key, + ) + if current and current["status"] == "done": + return current["response_status"], json.loads(current["response_body"]) + raise TimeoutError("concurrent idempotent request did not finish in time") diff --git a/app/images.py b/app/images.py new file mode 100644 index 0000000..5d6eb89 --- /dev/null +++ b/app/images.py @@ -0,0 +1,29 @@ +"""Server-side image validation: real format by content, not by the client's claim.""" +import io + +from PIL import Image, UnidentifiedImageError + +CONTENT_TYPE_TO_PIL_FORMAT = {"image/png": "PNG", "image/jpeg": "JPEG"} + + +class InvalidImage(Exception): + pass + + +def validate(data: bytes, declared_content_type: str, max_pixels: int) -> None: + """Raise InvalidImage if the bytes are not a genuine, within-budget PNG/JPEG.""" + expected_format = CONTENT_TYPE_TO_PIL_FORMAT.get(declared_content_type) + if expected_format is None: + raise InvalidImage("unsupported content type") + try: + with Image.open(io.BytesIO(data)) as img: + # Image.open() only parses the header; check the pixel budget + # BEFORE calling .load(), which is what actually decodes pixels. + if img.format != expected_format: + raise InvalidImage("declared content type does not match actual image format") + width, height = img.size + if width * height > max_pixels: + raise InvalidImage("image exceeds the decoded pixel budget") + img.load() # forces full decode; raises on truncated/corrupted data + except (UnidentifiedImageError, OSError, ValueError) as exc: + raise InvalidImage(str(exc)) from exc diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..ff04eae --- /dev/null +++ b/app/main.py @@ -0,0 +1,768 @@ +"""Messenger API — lab 5: durable queue + async attachment processing.""" +import asyncio +import base64 +import binascii +import json +import logging +import os +import re +import socket +import time +import uuid +from datetime import datetime, timezone + +import asyncpg +from fastapi import FastAPI, Header, Request +from fastapi.responses import JSONResponse, Response +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +import hashlib +import os + +from . import db, sessions, storage, images, broker, outbox, telemetry, metrics as app_metrics +from . import idempotency +from .errors import ( + ApiError, bad_request, conflict, not_found, payload_too_large, + unauthorized, unavailable, unsupported_media_type, +) +from .security import hash_password, verify_password + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +FILENAME_FORBIDDEN_RE = re.compile(r"[/\\\x00-\x1f]") +MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) +INSTANCE_ID = os.environ.get("INSTANCE_ID") or f"{socket.gethostname()}-{os.getpid()}" + +app = FastAPI(title="MTUSI Messenger — lab 6") +telemetry.init(os.environ.get("OTEL_SERVICE_NAME", "messenger-api"), app=app) +app_metrics.init() +logger = logging.getLogger("api") + +from opentelemetry import trace as _trace # noqa: E402 + +_tracer = _trace.get_tracer("messenger-api") + + +# --------------------------------------------------------------------------- +# Request/response plumbing: request id, instance id, canonical error shape. +# --------------------------------------------------------------------------- + +@app.middleware("http") +async def tagging_middleware(request: Request, call_next): + request_id = uuid.uuid4().hex + request.state.request_id = request_id + started = time.monotonic() + # Our own span wraps the whole inner chain (FastAPIInstrumentor's server + # span becomes its child). We deliberately log/record metrics INSIDE this + # span's context, not after call_next() returns: FastAPI's own span is + # already ended and its context detached by then, which would silently + # strip trace_id/span_id from the request-summary log line below. + with _tracer.start_as_current_span(f"http {request.method}") if _tracer else _noop_cm(): + try: + response = await call_next(request) + except ApiError as exc: + response = _error_response(exc, request_id) + except Exception: # noqa: BLE001 - never leak internals + response = _error_response(ApiError(500, "internal_error", "internal error"), request_id) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + route = request.scope.get("route") + route_template = route.path if route is not None else request.url.path + duration = time.monotonic() - started + app_metrics.record_http_request(request.method, route_template, response.status_code, duration) + level = logging.WARNING if response.status_code >= 500 else logging.INFO + logger.log(level, "%s %s -> %d in %.3fs", request.method, route_template, response.status_code, duration) + return response + + +class _noop_cm: + def __enter__(self): + return None + + def __exit__(self, *exc): + return False + + +def _error_response(exc: ApiError, request_id: str) -> JSONResponse: + headers = {} + if exc.status == 401: + path = "/api/v1/auth/sessions" + scheme = 'Basic realm="messenger", charset="UTF-8"' if _is_login_path(exc) else 'Bearer realm="messenger"' + headers["WWW-Authenticate"] = scheme + return JSONResponse( + status_code=exc.status, + content={"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}, + headers=headers, + ) + + +def _is_login_path(exc: ApiError) -> bool: + return getattr(exc, "login_path", False) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return _error_response(exc, getattr(request.state, "request_id", uuid.uuid4().hex)) + + +def iso(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def user_json(record) -> dict: + return { + "id": str(record["id"]), + "username": record["username"], + "display_name": record["display_name"], + "created_at": iso(record["created_at"]), + } + + +def chat_json(record, member_ids: list[uuid.UUID]) -> dict: + return { + "id": str(record["id"]), + "title": record["title"], + "member_ids": [str(m) for m in member_ids], + "created_at": iso(record["created_at"]), + } + + +def message_json(record, attachment_ids: list[uuid.UUID]) -> dict: + return { + "id": str(record["id"]), + "chat_id": str(record["chat_id"]), + "sender_id": str(record["sender_id"]), + "text": record["text"], + "attachment_ids": [str(a) for a in attachment_ids], + "created_at": iso(record["created_at"]), + } + + +def attachment_json(record, variant_rows=()) -> dict: + return { + "id": str(record["id"]), + "chat_id": str(record["chat_id"]), + "owner_id": str(record["owner_id"]), + "filename": record["filename"], + "content_type": record["content_type"], + "size_bytes": record["size_bytes"], + "sha256": record["sha256"], + "status": record["status"], + "variants": [variant_json(v) for v in variant_rows], + "error_code": record["error_code"], + "created_at": iso(record["created_at"]), + "updated_at": iso(record["updated_at"]), + } + + +def variant_json(record) -> dict: + return { + "name": record["name"], + "content_type": record["content_type"], + "size_bytes": record["size_bytes"], + "width": record["width"], + "height": record["height"], + "sha256": record["sha256"], + } + + +# --------------------------------------------------------------------------- +# Cursors: opaque base64url(json) of the keyset tuple, scoped per resource. +# --------------------------------------------------------------------------- + +def encode_cursor(payload: dict) -> str: + raw = json.dumps(payload, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def decode_cursor(value: str) -> dict: + try: + padded = value + "=" * (-len(value) % 4) + raw = base64.urlsafe_b64decode(padded.encode()) + return json.loads(raw) + except (binascii.Error, ValueError, UnicodeDecodeError): + raise bad_request("malformed cursor") + + +# --------------------------------------------------------------------------- +# Auth: Basic only on POST /auth/sessions, Bearer everywhere else protected. +# --------------------------------------------------------------------------- + +async def require_bearer(authorization: str | None) -> asyncpg.Record: + if not authorization or not authorization.startswith("Bearer "): + raise unauthorized() + token = authorization[len("Bearer "):].strip() + if not token: + raise unauthorized() + try: + user_id = await sessions.resolve(token) + except Exception: # noqa: BLE001 - dependency failure must not grant access + raise unavailable("session store unavailable") + if user_id is None: + raise unauthorized() + async with db.pool().acquire() as conn: + record = await conn.fetchrow("SELECT * FROM users WHERE id=$1", user_id) + if record is None: + raise unauthorized() + return record + + +def parse_basic(authorization: str | None) -> tuple[str, str]: + err = ApiError(401, "unauthorized", "Basic credentials required") + err.login_path = True + if not authorization or not authorization.startswith("Basic "): + raise err + try: + decoded = base64.b64decode(authorization[len("Basic "):].strip()).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + raise err + if ":" not in decoded: + raise err + username, password = decoded.split(":", 1) + return username, password + + +# --------------------------------------------------------------------------- +# Request bodies (additionalProperties: false in the contract -> extra="forbid"). +# --------------------------------------------------------------------------- + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(max_length=4000) # required key, "" allowed if attachments are present + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + +async def parse_body(request: Request, model): + try: + raw = await request.json() + except json.JSONDecodeError: + raise bad_request("malformed JSON") + try: + return model.model_validate(raw) + except ValidationError as exc: + raise bad_request(str(exc.errors()[0]["msg"]) if exc.errors() else "invalid body") + + +# --------------------------------------------------------------------------- +# Health +# --------------------------------------------------------------------------- + +@app.get("/health/live") +async def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +async def health_ready(): + if await db.ping() and await sessions.ping() and await storage.ping() and await broker.ping(): + return {"status": "ok"} + raise unavailable("database, session store, object storage or broker not ready") + + +# --------------------------------------------------------------------------- +# Users +# --------------------------------------------------------------------------- + +@app.post("/api/v1/users", status_code=201) +async def register_user(request: Request): + body = await parse_body(request, CreateUser) + if not USERNAME_RE.match(body.username): + raise bad_request("username must match ^[a-z0-9_]{3,32}$") + if not body.display_name.strip(): + raise bad_request("display_name must not be blank") + password_hash = hash_password(body.password) + try: + async with db.pool().acquire() as conn: + record = await conn.fetchrow( + """ + INSERT INTO users (username, display_name, password_hash) + VALUES ($1, $2, $3) + RETURNING id, username, display_name, created_at + """, + body.username, + body.display_name, + password_hash, + ) + except asyncpg.UniqueViolationError: + raise conflict("username already taken") + return JSONResponse(status_code=201, content=user_json(record)) + + +@app.get("/api/v1/users/me") +async def current_user(authorization: str | None = Header(default=None)): + record = await require_bearer(authorization) + return user_json(record) + + +# --------------------------------------------------------------------------- +# Sessions +# --------------------------------------------------------------------------- + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(authorization: str | None = Header(default=None)): + username, password = parse_basic(authorization) + async with db.pool().acquire() as conn: + record = await conn.fetchrow("SELECT * FROM users WHERE username=$1", username) + err = ApiError(401, "unauthorized", "invalid credentials") + err.login_path = True + if record is None or not verify_password(password, record["password_hash"]): + raise err + try: + token, expires_at = await sessions.create(record["id"]) + except Exception: # noqa: BLE001 + raise unavailable("session store unavailable") + return JSONResponse( + status_code=201, + content={ + "token": token, + "token_type": "Bearer", + "expires_at": iso(expires_at), + "user": user_json(record), + }, + ) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def revoke_session(authorization: str | None = Header(default=None)): + await require_bearer(authorization) # validates the token is currently live + token = authorization[len("Bearer "):].strip() + try: + await sessions.revoke(token) + except Exception: # noqa: BLE001 + raise unavailable("session store unavailable") + return Response(status_code=204) + + +# --------------------------------------------------------------------------- +# Chats +# --------------------------------------------------------------------------- + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request, authorization: str | None = Header(default=None)): + actor = await require_bearer(authorization) + body = await parse_body(request, CreateChat) + try: + member_ids = [uuid.UUID(m) for m in body.member_ids] + except ValueError: + raise bad_request("member_ids must be UUIDs") + if len(set(member_ids)) != len(member_ids): + raise bad_request("member_ids must not contain duplicates") + if actor["id"] not in member_ids: + raise bad_request("caller must be a member of the chat") + async with db.pool().acquire() as conn: + async with conn.transaction(): + existing = await conn.fetch( + "SELECT id FROM users WHERE id = ANY($1::uuid[])", member_ids + ) + if len(existing) != len(member_ids): + raise bad_request("unknown member in member_ids") + chat = await conn.fetchrow( + "INSERT INTO chats (title) VALUES ($1) RETURNING id, title, created_at", + body.title, + ) + await conn.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(chat["id"], m) for m in member_ids], + ) + return JSONResponse(status_code=201, content=chat_json(chat, member_ids)) + + +@app.get("/api/v1/chats") +async def list_chats( + authorization: str | None = Header(default=None), + limit: int = 50, + cursor: str | None = None, +): + actor = await require_bearer(authorization) + if not (1 <= limit <= 100): + raise bad_request("limit must be between 1 and 100") + after_created_at = after_id = None + if cursor: + payload = decode_cursor(cursor) + try: + after_created_at = datetime.fromisoformat(payload["created_at"]) + after_id = uuid.UUID(payload["id"]) + except (KeyError, ValueError): + raise bad_request("malformed cursor") + async with db.pool().acquire() as conn: + rows = await conn.fetch( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = $1 + AND ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3)) + ORDER BY c.created_at ASC, c.id ASC + LIMIT $4 + """, + actor["id"], + after_created_at, + after_id, + limit + 1, + ) + has_more = len(rows) > limit + rows = rows[:limit] + items = [] + for row in rows: + members = await conn.fetch("SELECT user_id FROM chat_members WHERE chat_id=$1", row["id"]) + items.append(chat_json(row, [m["user_id"] for m in members])) + next_cursor = None + if has_more: + last = rows[-1] + next_cursor = encode_cursor({"created_at": iso(last["created_at"]), "id": str(last["id"])}) + return {"items": items, "next_cursor": next_cursor} + + +async def _load_chat_if_member(conn, chat_id: uuid.UUID, user_id: uuid.UUID): + chat = await conn.fetchrow("SELECT id, title, created_at FROM chats WHERE id=$1", chat_id) + if chat is None: + raise not_found("chat not found") + membership = await conn.fetchrow( + "SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2", chat_id, user_id + ) + if membership is None: + raise not_found("chat not found") + return chat + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, authorization: str | None = Header(default=None)): + actor = await require_bearer(authorization) + try: + chat_uuid = uuid.UUID(chat_id) + except ValueError: + raise bad_request("chat_id must be a UUID") + async with db.pool().acquire() as conn: + chat = await _load_chat_if_member(conn, chat_uuid, actor["id"]) + members = await conn.fetch("SELECT user_id FROM chat_members WHERE chat_id=$1", chat_uuid) + return chat_json(chat, [m["user_id"] for m in members]) + + +# --------------------------------------------------------------------------- +# Messages +# --------------------------------------------------------------------------- + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message( + chat_id: str, + request: Request, + authorization: str | None = Header(default=None), + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), +): + # Session/access are re-checked on every call, including idempotent replays. + actor = await require_bearer(authorization) + try: + chat_uuid = uuid.UUID(chat_id) + except ValueError: + raise bad_request("chat_id must be a UUID") + if not idempotency_key or not IDEMPOTENCY_KEY_RE.match(idempotency_key): + raise bad_request("Idempotency-Key header is required and must match ^[A-Za-z0-9._:-]{1,128}$") + body = await parse_body(request, CreateMessage) + try: + attachment_uuids = [uuid.UUID(a) for a in body.attachment_ids] + except ValueError: + raise bad_request("attachment_ids must be UUIDs") + if len(set(attachment_uuids)) != len(attachment_uuids): + raise bad_request("attachment_ids must not contain duplicates") + if not body.text.strip() and not attachment_uuids: + raise bad_request("text must not be blank unless attachments are present") + + async def perform(): + record = await conn.fetchrow( + """ + INSERT INTO messages (chat_id, sender_id, text) + VALUES ($1, $2, $3) + RETURNING id, chat_id, sender_id, text, created_at + """, + chat_uuid, + actor["id"], + body.text, + ) + if attachment_uuids: + owned = await conn.fetch( + "SELECT id, status FROM attachments WHERE chat_id=$1 AND id = ANY($2::uuid[])", + chat_uuid, attachment_uuids, + ) + if len(owned) != len(attachment_uuids): + raise not_found("one or more attachment_ids are not available in this chat") + if any(r["status"] == "failed" for r in owned): + raise conflict("one or more attachment_ids reference a failed attachment") + await conn.executemany( + "INSERT INTO message_attachments (message_id, attachment_id) VALUES ($1, $2)", + [(record["id"], a) for a in attachment_uuids], + ) + return 201, message_json(record, attachment_uuids) + + async with db.pool().acquire() as conn: + await _load_chat_if_member(conn, chat_uuid, actor["id"]) + async with conn.transaction(): + try: + status_code, payload = await idempotency.run_once( + conn, + actor["id"], + "POST", + request.url.path, + idempotency_key, + {"text": body.text, "attachment_ids": sorted(body.attachment_ids)}, + perform, + ) + except idempotency.IdempotencyConflict: + raise conflict("Idempotency-Key reused with a different request body") + return JSONResponse(status_code=status_code, content=payload) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages( + chat_id: str, + authorization: str | None = Header(default=None), + limit: int = 50, + cursor: str | None = None, +): + actor = await require_bearer(authorization) + try: + chat_uuid = uuid.UUID(chat_id) + except ValueError: + raise bad_request("chat_id must be a UUID") + if not (1 <= limit <= 100): + raise bad_request("limit must be between 1 and 100") + after_created_at = after_id = None + if cursor: + payload = decode_cursor(cursor) + if payload.get("chat_id") != chat_id: + raise bad_request("cursor does not belong to this chat") + try: + after_created_at = datetime.fromisoformat(payload["created_at"]) + after_id = uuid.UUID(payload["id"]) + except (KeyError, ValueError): + raise bad_request("malformed cursor") + async with db.pool().acquire() as conn: + await _load_chat_if_member(conn, chat_uuid, actor["id"]) + rows = await conn.fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3)) + ORDER BY created_at ASC, id ASC + LIMIT $4 + """, + chat_uuid, + after_created_at, + after_id, + limit + 1, + ) + has_more = len(rows) > limit + rows = rows[:limit] + items = [] + for r in rows: + attached = await conn.fetch( + "SELECT attachment_id FROM message_attachments WHERE message_id=$1", r["id"] + ) + items.append(message_json(r, [a["attachment_id"] for a in attached])) + next_cursor = None + if has_more: + last = rows[-1] + next_cursor = encode_cursor( + {"chat_id": chat_id, "created_at": iso(last["created_at"]), "id": str(last["id"])} + ) + return {"items": items, "next_cursor": next_cursor} + + +# --------------------------------------------------------------------------- +# Attachments +# --------------------------------------------------------------------------- + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str, + authorization: str | None = Header(default=None), +): + actor = await require_bearer(authorization) + try: + chat_uuid = uuid.UUID(chat_id) + except ValueError: + raise bad_request("chat_id must be a UUID") + if not (1 <= len(filename) <= 255) or FILENAME_FORBIDDEN_RE.search(filename): + raise bad_request("filename must be 1-255 characters with no path separators or control characters") + content_type = request.headers.get("content-type", "") + if content_type not in ("image/png", "image/jpeg"): + raise unsupported_media_type("only image/png and image/jpeg are accepted") + + # Enforce the byte budget BEFORE reading an unbounded body into memory. + data = bytearray() + async for chunk in request.stream(): + data.extend(chunk) + if len(data) > MAX_UPLOAD_BYTES: + raise payload_too_large(f"upload exceeds {MAX_UPLOAD_BYTES} bytes") + data = bytes(data) + if not data: + raise bad_request("empty upload body") + + try: + images.validate(data, content_type, MAX_IMAGE_PIXELS) + except images.InvalidImage as exc: + raise unsupported_media_type(str(exc)) + + digest = hashlib.sha256(data).hexdigest() + # Staging IS the final S3 prefix here (same bucket/prefix the worker later + # reads from) -- there is no separate move step, which is an honest + # simplification the contract explicitly allows when staging is already S3. + object_key = storage.new_object_key() + + pool = db.pool() + async with pool.acquire() as conn: + await _load_chat_if_member(conn, chat_uuid, actor["id"]) + try: + await storage.put_object(object_key, data, content_type) + except Exception: # noqa: BLE001 + raise unavailable("object storage unavailable") + async with conn.transaction(): + record = await conn.fetchrow( + """ + INSERT INTO attachments (chat_id, owner_id, filename, content_type, size_bytes, sha256, status, s3_key) + VALUES ($1, $2, $3, $4, $5, $6, 'queued', $7) + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at + """, + chat_uuid, actor["id"], filename, content_type, len(data), digest, object_key, + ) + job = await outbox.enqueue(conn, record["id"]) + # Best-effort immediate publish; if this process dies right here, the + # outbox relay (any replica) still publishes it from the committed row. + await outbox.publish_best_effort(pool, job) + return JSONResponse(status_code=202, content=attachment_json(record)) + + +async def _load_attachment_if_member(conn, attachment_id: uuid.UUID, user_id: uuid.UUID): + record = await conn.fetchrow( + """ + SELECT a.* FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id + WHERE a.id = $1 AND cm.user_id = $2 + """, + attachment_id, user_id, + ) + if record is None: + raise not_found("attachment not found") + return record + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(attachment_id: str, authorization: str | None = Header(default=None)): + actor = await require_bearer(authorization) + try: + att_uuid = uuid.UUID(attachment_id) + except ValueError: + raise bad_request("attachment_id must be a UUID") + async with db.pool().acquire() as conn: + record = await _load_attachment_if_member(conn, att_uuid, actor["id"]) + variant_rows = await conn.fetch( + "SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants WHERE attachment_id=$1", + att_uuid, + ) + return attachment_json(record, variant_rows) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def get_attachment_content( + attachment_id: str, + authorization: str | None = Header(default=None), + variant: str = "original", +): + actor = await require_bearer(authorization) + try: + att_uuid = uuid.UUID(attachment_id) + except ValueError: + raise bad_request("attachment_id must be a UUID") + if variant not in ("original", "thumbnail", "crop", "optimized"): + raise bad_request("unknown variant") + async with db.pool().acquire() as conn: + record = await _load_attachment_if_member(conn, att_uuid, actor["id"]) + if record["status"] in ("queued", "processing"): + raise conflict("attachment is not ready yet") + if record["status"] == "failed": + raise conflict("attachment processing failed") + if variant == "original": + s3_key, content_type = record["s3_key"], record["content_type"] + else: + variant_row = await conn.fetchrow( + "SELECT s3_key, content_type FROM attachment_variants WHERE attachment_id=$1 AND name=$2", + att_uuid, variant, + ) + if variant_row is None: + raise not_found(f"variant {variant} is not available for this attachment") + s3_key, content_type = variant_row["s3_key"], variant_row["content_type"] + try: + data = await storage.get_object(s3_key) + except Exception: # noqa: BLE001 + raise unavailable("object storage unavailable") + return Response(content=data, media_type=content_type) + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + +@app.on_event("startup") +async def on_startup(): + database_url = _read_secret("DATABASE_URL_FILE", "DATABASE_URL") + cache_url = _read_secret("CACHE_URL_FILE", "CACHE_URL") + await db.init_pool(database_url) + sessions.init(cache_url) + storage.init( + endpoint=os.environ["S3_ENDPOINT"], + region=os.environ.get("S3_REGION", "us-east-1"), + access_key=_read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"), + secret_key=_read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"), + bucket=os.environ.get("S3_BUCKET", "messenger-attachments"), + ) + await storage.ensure_bucket() + broker.init(cache_url) + await broker.ensure_group() + app.state.outbox_relay_task = asyncio.create_task(outbox.relay_loop(db.pool())) + + loop = asyncio.get_event_loop() + + def _db_pool_state(): + pool = db.pool() + used = pool.get_size() - pool.get_idle_size() + return used, pool.get_idle_size() + + def _queue_state(): + # Measured from the API, not just the worker: the backlog must stay + # observable even while the worker itself is down (that IS the incident). + return asyncio.run_coroutine_threadsafe(broker.depth_and_oldest_age_seconds(), loop).result(timeout=5) + + app_metrics.observable_gauges(db_pool_getter=_db_pool_state, queue_depth_getter=_queue_state) + + +@app.on_event("shutdown") +async def on_shutdown(): + app.state.outbox_relay_task.cancel() + await db.close_pool() + await sessions.close() + + +def _read_secret(file_env: str, inline_env: str) -> str: + path = os.environ.get(file_env) + if path and os.path.isfile(path): + return open(path, encoding="utf-8").read().strip() + value = os.environ.get(inline_env) + if not value: + raise RuntimeError(f"{file_env} or {inline_env} must be set") + return value diff --git a/app/metrics.py b/app/metrics.py new file mode 100644 index 0000000..0e4c292 --- /dev/null +++ b/app/metrics.py @@ -0,0 +1,82 @@ +"""Custom business/HTTP metrics per contracts/observability.md. + +Every label here is a fixed, low-cardinality string (route template, job +type, outcome, queue name) -- never a UUID (user_id/chat_id/message_id/ +attachment_id/request_id never become a label value). +""" +from . import telemetry + +_http_requests_total = None +_http_request_duration_seconds = None +_jobs_processed_total = None +_job_processing_duration_seconds = None +_time_to_ready_seconds = None + + +def init() -> None: + global _http_requests_total, _http_request_duration_seconds + global _jobs_processed_total, _job_processing_duration_seconds, _time_to_ready_seconds + m = telemetry.meter() + _http_requests_total = m.create_counter( + "http_requests_total", description="HTTP requests by method/route template/status class" + ) + _http_request_duration_seconds = m.create_histogram( + "http_request_duration_seconds", unit="s", description="HTTP request duration" + ) + _jobs_processed_total = m.create_counter( + "jobs_processed_total", description="Attachment jobs by job type and outcome" + ) + _job_processing_duration_seconds = m.create_histogram( + "job_processing_duration_seconds", unit="s", description="Worker job processing duration" + ) + _time_to_ready_seconds = m.create_histogram( + "time_to_ready_seconds", unit="s", + description="End-to-end time from durable accept (202) to ready", + ) + + +def record_http_request(method: str, route: str, status_code: int, duration_seconds: float) -> None: + status_class = f"{status_code // 100}xx" + attrs = {"method": method, "route": route, "status_class": status_class} + _http_requests_total.add(1, attrs) + _http_request_duration_seconds.record(duration_seconds, attrs) + + +def record_job(job_type: str, outcome: str, duration_seconds: float) -> None: + attrs = {"job_type": job_type, "outcome": outcome} + _jobs_processed_total.add(1, attrs) + _job_processing_duration_seconds.record(duration_seconds, attrs) + + +def record_time_to_ready(seconds: float) -> None: + _time_to_ready_seconds.record(seconds, {"job_type": "attachment.process.v1"}) + + +def observable_gauges(db_pool_getter=None, queue_depth_getter=None, worker_slots_getter=None) -> None: + """Register async-callback gauges; called once at startup with closures + that read live state (DB pool, broker queue depth, worker concurrency).""" + m = telemetry.meter() + if db_pool_getter: + def _db_pool_cb(options): + used, idle = db_pool_getter() + from opentelemetry.metrics import Observation + yield Observation(used, {"state": "used"}) + yield Observation(idle, {"state": "idle"}) + m.create_observable_gauge("db_pool_connections", callbacks=[_db_pool_cb], description="asyncpg pool connections") + + if queue_depth_getter: + def _queue_cb(options): + from opentelemetry.metrics import Observation + depth, oldest_age = queue_depth_getter() + yield Observation(depth, {"queue": "attachment_jobs", "metric": "depth"}) + if oldest_age is not None: + yield Observation(oldest_age, {"queue": "attachment_jobs", "metric": "oldest_age_seconds"}) + m.create_observable_gauge("queue_backlog", callbacks=[_queue_cb], description="Pending job count and oldest pending age") + + if worker_slots_getter: + def _worker_cb(options): + from opentelemetry.metrics import Observation + busy, capacity = worker_slots_getter() + yield Observation(busy, {"state": "busy"}) + yield Observation(capacity, {"state": "capacity"}) + m.create_observable_gauge("worker_concurrency_slots", callbacks=[_worker_cb], description="Worker busy vs capacity slots") diff --git a/app/outbox.py b/app/outbox.py new file mode 100644 index 0000000..e306b67 --- /dev/null +++ b/app/outbox.py @@ -0,0 +1,96 @@ +"""Transactional outbox: bridges the PostgreSQL COMMIT and the broker publish. + +enqueue() writes the job row in the SAME transaction as the attachments +INSERT, so either both happen or neither does. The caller then makes a +best-effort immediate publish; if the process crashes between COMMIT and +that publish, the row is still there with published_at IS NULL, and the +background relay (any API replica) picks it up and publishes it later -- +no job is silently lost, and none is invented without a committed row. +""" +import asyncio +import datetime as dt +import logging +import uuid + +import asyncpg + +from . import broker, telemetry + +logger = logging.getLogger("app.outbox") + +RELAY_INTERVAL_SECONDS = 2 +RELAY_GRACE_SECONDS = 1 # don't race the enqueuing request's own immediate publish + + +async def enqueue(conn: asyncpg.Connection, attachment_id: uuid.UUID) -> dict: + job_id = uuid.uuid4() + await conn.execute( + "INSERT INTO job_outbox (id, attachment_id, job_id, pipeline_version, attempt) " + "VALUES (gen_random_uuid(), $1, $2, 1, 1)", + attachment_id, job_id, + ) + job = { + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": 1, + "attempt": 1, + "created_at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"), + } + # W3C trace context: lets the worker's consumer span link back to this + # request's producer span (contracts/observability.md: sequenced upload-> + # publish->consume->S3/DB chain). + telemetry.inject_context(job) + return job + + +async def publish_best_effort(pool: asyncpg.Pool, job: dict) -> None: + """Try to publish right away; on failure, leave it for the relay. Never + raises -- a broker hiccup here must not fail the already-committed upload.""" + try: + await broker.publish(job) + async with pool.acquire() as conn: + await conn.execute( + "UPDATE job_outbox SET published_at=now() WHERE job_id=$1", uuid.UUID(job["job_id"]) + ) + except Exception: # noqa: BLE001 + logger.warning("immediate publish failed for job %s; relay will retry", job["job_id"]) + + +async def relay_once(pool: asyncpg.Pool) -> int: + async with pool.acquire() as conn: + rows = await conn.fetch( + """ + SELECT id, attachment_id, job_id, pipeline_version, attempt, created_at + FROM job_outbox + WHERE published_at IS NULL AND created_at < now() - ($1 * interval '1 second') + ORDER BY created_at + LIMIT 50 + """, + RELAY_GRACE_SECONDS, + ) + for row in rows: + job = { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": row["pipeline_version"], + "attempt": row["attempt"], + "created_at": row["created_at"].astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z"), + } + await broker.publish(job) + await conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=$1", row["id"]) + return len(rows) + + +async def relay_loop(pool: asyncpg.Pool) -> None: + while True: + try: + await relay_once(pool) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - the relay must survive transient DB/broker hiccups + logger.exception("outbox relay tick failed") + await asyncio.sleep(RELAY_INTERVAL_SECONDS) diff --git a/app/pipeline.py b/app/pipeline.py new file mode 100644 index 0000000..fc277cb --- /dev/null +++ b/app/pipeline.py @@ -0,0 +1,57 @@ +"""Worker-side variant generation: thumbnail (grade 3), crop + optimized (grade 4).""" +import io + +from PIL import Image, ImageOps + + +def _normalize(img: Image.Image) -> Image.Image: + """Apply EXIF orientation, then flatten any alpha onto white.""" + img = ImageOps.exif_transpose(img) + if img.mode in ("RGBA", "LA", "P"): + rgba = img.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.split()[-1]) + return background + return img.convert("RGB") + + +def thumbnail(data: bytes) -> tuple[bytes, int, int]: + """JPEG, contained within 256x256, never upscaled, rounded down, min 1px.""" + with Image.open(io.BytesIO(data)) as raw: + img = _normalize(raw) + scale = min(256 / img.width, 256 / img.height, 1.0) + width = max(1, int(img.width * scale)) + height = max(1, int(img.height * scale)) + resized = img.resize((width, height), Image.LANCZOS) + buf = io.BytesIO() + resized.save(buf, "JPEG", quality=85) + return buf.getvalue(), width, height + + +def crop(data: bytes) -> tuple[bytes, int, int]: + """JPEG, centered 128x128 square; small sources may be upscaled.""" + with Image.open(io.BytesIO(data)) as raw: + img = _normalize(raw) + scale = 128 / min(img.width, img.height) + width = max(128, round(img.width * scale)) + height = max(128, round(img.height * scale)) + resized = img.resize((width, height), Image.LANCZOS) + left = (width - 128) // 2 + top = (height - 128) // 2 + cropped = resized.crop((left, top, left + 128, top + 128)) + buf = io.BytesIO() + cropped.save(buf, "JPEG", quality=85) + return buf.getvalue(), 128, 128 + + +def optimized(data: bytes) -> tuple[bytes, int, int]: + """WebP, contained within 1280x1280, never upscaled, EXIF stripped.""" + with Image.open(io.BytesIO(data)) as raw: + img = _normalize(raw) + scale = min(1280 / img.width, 1280 / img.height, 1.0) + width = max(1, int(img.width * scale)) + height = max(1, int(img.height * scale)) + resized = img.resize((width, height), Image.LANCZOS) + buf = io.BytesIO() + resized.save(buf, "WEBP", quality=80) # save() never copies source EXIF for WEBP + return buf.getvalue(), width, height diff --git a/app/security.py b/app/security.py new file mode 100644 index 0000000..536e287 --- /dev/null +++ b/app/security.py @@ -0,0 +1,16 @@ +"""Password KDF (Argon2id) — never store or compare plaintext passwords.""" +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError, InvalidHashError + +_hasher = PasswordHasher() + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password: str, password_hash: str) -> bool: + try: + return _hasher.verify(password_hash, password) + except (VerifyMismatchError, InvalidHashError): + return False diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 0000000..4eb4a9c --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,54 @@ +"""Opaque Bearer sessions backed by Redis/Valkey, TTL-bound, AOF-persistent.""" +import json +import os +import secrets +import uuid +from datetime import datetime, timedelta, timezone + +import redis.asyncio as redis + +_client: redis.Redis | None = None +TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + + +def init(url: str) -> None: + global _client + _client = redis.from_url(url, decode_responses=True, socket_timeout=3, socket_connect_timeout=3) + + +async def close() -> None: + if _client is not None: + await _client.close() + + +def client() -> redis.Redis: + if _client is None: + raise RuntimeError("session store is not initialized") + return _client + + +async def ping() -> bool: + try: + return bool(await client().ping()) + except Exception: # noqa: BLE001 - readiness probe + return False + + +async def create(user_id: uuid.UUID) -> tuple[str, datetime]: + token = secrets.token_urlsafe(32) # 256 bits of entropy, opaque and non-guessable + expires_at = datetime.now(timezone.utc) + timedelta(seconds=TTL_SECONDS) + payload = json.dumps({"user_id": str(user_id), "expires_at": expires_at.isoformat()}) + await client().set(f"session:{token}", payload, ex=TTL_SECONDS) + return token, expires_at + + +async def resolve(token: str) -> uuid.UUID | None: + raw = await client().get(f"session:{token}") + if raw is None: + return None + data = json.loads(raw) + return uuid.UUID(data["user_id"]) + + +async def revoke(token: str) -> None: + await client().delete(f"session:{token}") diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 0000000..34bd27b --- /dev/null +++ b/app/storage.py @@ -0,0 +1,63 @@ +"""Private S3-compatible object storage for attachment bytes (MinIO behind the scenes). + +boto3 is synchronous; calls are pushed to a thread so the event loop is not +blocked by S3 I/O. Object keys are server-generated UUIDs, never the +user-supplied filename (filename is display-only, see contracts/README.md). +""" +import asyncio +import io +import uuid + +import boto3 +from botocore.client import Config +from botocore.exceptions import ClientError + +_client = None +_bucket: str | None = None + + +def init(endpoint: str, region: str, access_key: str, secret_key: str, bucket: str) -> None: + global _client, _bucket + _bucket = bucket + _client = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=Config(signature_version="s3v4", retries={"max_attempts": 2}), + ) + + +async def ensure_bucket() -> None: + """The API identity only has Read/Write/List on this one bucket, not + bucket-admin rights, so CreateBucket is not an option. SeaweedFS (like S3) + auto-creates the bucket on the first successful PutObject, so writing an + empty marker object both creates the bucket and proves write access.""" + await asyncio.to_thread(_client.put_object, Bucket=_bucket, Key=".bootstrap", Body=b"") + + +async def ping() -> bool: + try: + await asyncio.to_thread(_client.head_object, Bucket=_bucket, Key=".bootstrap") + return True + except Exception: # noqa: BLE001 - readiness probe + return False + + +def new_object_key() -> str: + return f"attachments/{uuid.uuid4().hex}" + + +async def put_object(key: str, data: bytes, content_type: str) -> None: + await asyncio.to_thread( + _client.put_object, Bucket=_bucket, Key=key, Body=data, ContentType=content_type + ) + + +async def get_object(key: str) -> bytes: + def _get(): + resp = _client.get_object(Bucket=_bucket, Key=key) + return resp["Body"].read() + + return await asyncio.to_thread(_get) diff --git a/app/telemetry.py b/app/telemetry.py new file mode 100644 index 0000000..d5fe6df --- /dev/null +++ b/app/telemetry.py @@ -0,0 +1,133 @@ +"""OpenTelemetry wiring: traces/metrics via OTLP to the Collector, JSON logs +on stdout correlated with the active trace/span. Shared by the API and the +worker -- only service.name differs, set via OTEL_SERVICE_NAME per process. + +No user_id/chat_id/message_id/attachment_id/request_id ever becomes a metric +label (contracts/observability.md): job/route identifiers used as labels are +fixed strings (job type, route template), never raw UUIDs. +""" +import json +import logging +import os +import sys +import time + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.redis import RedisInstrumentor +from opentelemetry.propagate import extract, inject +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.semconv.resource import ResourceAttributes + +_meter = None +_initialized = False + + +class JsonLogFormatter(logging.Formatter): + """timestamp, severity, service.name, event, trace_id/span_id when a span is active.""" + + def __init__(self, service_name: str): + super().__init__() + self.service_name = service_name + + def format(self, record: logging.LogRecord) -> str: + payload = { + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + "Z", + "severity": record.levelname, + "service.name": self.service_name, + "event": record.getMessage(), + } + span = trace.get_current_span() + ctx = span.get_span_context() if span else None + if ctx and ctx.is_valid: + payload["trace_id"] = format(ctx.trace_id, "032x") + payload["span_id"] = format(ctx.span_id, "016x") + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + return json.dumps(payload, ensure_ascii=False) + + +def init(service_name: str, app=None) -> None: + """Call once per process (API or worker). `app` is the FastAPI instance, if any.""" + global _meter, _initialized + if _initialized: + return + _initialized = True + + endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318") + resource = Resource.create( + { + ResourceAttributes.SERVICE_NAME: service_name, + ResourceAttributes.SERVICE_VERSION: os.environ.get("SERVICE_VERSION", "0.1.0"), + ResourceAttributes.SERVICE_INSTANCE_ID: os.environ.get("INSTANCE_ID", "unknown"), + "deployment.environment.name": os.environ.get("DEPLOYMENT_ENVIRONMENT", "lab"), + } + ) + + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint + "/v1/traces")) + ) + trace.set_tracer_provider(tracer_provider) + + meter_provider = MeterProvider( + resource=resource, + metric_readers=[ + PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=endpoint + "/v1/metrics"), export_interval_millis=5000 + ) + ], + ) + metrics.set_meter_provider(meter_provider) + _meter = metrics.get_meter(service_name) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint + "/v1/logs")) + ) + set_logger_provider(logger_provider) + + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setFormatter(JsonLogFormatter(service_name)) + otlp_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + root = logging.getLogger() + root.handlers = [stdout_handler, otlp_handler] + root.setLevel(logging.INFO) + + if app is not None: + FastAPIInstrumentor.instrument_app(app, tracer_provider=tracer_provider, meter_provider=meter_provider) + + # Dedicated DB/Redis client spans: lets a slow or failing query show up as + # its own child span with its own duration/error, distinct from a slow + # application handler that spends its time elsewhere (contracts/ + # observability.md: distinguish DB failure from a slow HTTP handler). + AsyncPGInstrumentor().instrument(tracer_provider=tracer_provider) + RedisInstrumentor().instrument(tracer_provider=tracer_provider) + + +def meter(): + if _meter is None: + raise RuntimeError("telemetry.init() was not called") + return _meter + + +def inject_context(carrier: dict) -> dict: + """W3C traceparent/tracestate into a job payload before publishing to the broker.""" + inject(carrier) + return carrier + + +def extract_context(carrier: dict): + """Build a Context from a job payload's traceparent/tracestate for the consumer span.""" + return extract(carrier) diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 0000000..a069fbe --- /dev/null +++ b/app/worker.py @@ -0,0 +1,248 @@ +"""Standalone worker process: consumes attachment.process.v1 jobs, generates +variants, writes results idempotently. Runs as its own container/command, +separate from the API (contracts/attachment-states.md requires this split). +""" +import asyncio +import datetime as dt +import hashlib +import logging +import os +import random +import socket +import time +import uuid + +from opentelemetry import trace +from opentelemetry.trace import SpanKind + +from . import broker, db, pipeline, storage, telemetry +from . import metrics as app_metrics + +logger = logging.getLogger("worker") + +CONSUMER_NAME = os.environ.get("INSTANCE_ID") or f"{socket.gethostname()}-{os.getpid()}" +MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) +JOB_TIMEOUT_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) +RETRY_SCAN_INTERVAL_SECONDS = 1 +WORKER_CAPACITY = 1 # this process handles one job at a time (see REPORT.md limitations) + +_active_jobs = 0 +_tracer = None + + +def read_secret(file_env: str, inline_env: str) -> str: + path = os.environ.get(file_env) + if path and os.path.isfile(path): + return open(path, encoding="utf-8").read().strip() + value = os.environ.get(inline_env) + if not value: + raise RuntimeError(f"{file_env} or {inline_env} must be set") + return value + + +async def process_job(job: dict) -> None: + global _active_jobs + ctx = telemetry.extract_context(job) if job.get("traceparent") else None + span_cm = _tracer.start_as_current_span("process_job", context=ctx, kind=SpanKind.CONSUMER) if _tracer else None + if span_cm: + span_cm.__enter__() + started = time.monotonic() + _active_jobs += 1 + try: + await _process_job_inner(job, started) + finally: + _active_jobs -= 1 + if span_cm: + span_cm.__exit__(None, None, None) + + +async def _process_job_inner(job: dict, started: float) -> None: + attachment_id = uuid.UUID(job["attachment_id"]) + job_id = uuid.UUID(job["job_id"]) + attempt = job["attempt"] + + async with db.pool().acquire() as conn: + record = await conn.fetchrow("SELECT * FROM attachments WHERE id=$1", attachment_id) + if record is None: + logger.warning("job %s references missing attachment %s; dropping", job_id, attachment_id) + return + if record["status"] in ("ready", "failed"): + # Redelivery of an already-finished job, or a stale duplicate racing + # a newer attempt: never roll a finished result backwards. + logger.info("attachment %s already %s; no-op", attachment_id, record["status"]) + app_metrics.record_job("attachment.process.v1", "duplicate_noop", time.monotonic() - started) + return + + async with db.pool().acquire() as conn: + await conn.execute( + "UPDATE attachments SET status='processing', updated_at=now() WHERE id=$1 AND status != 'ready'", + attachment_id, + ) + + content_types = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"} + try: + original = await storage.get_object(record["s3_key"]) + generated = { + "thumbnail": pipeline.thumbnail(original), + "crop": pipeline.crop(original), + "optimized": pipeline.optimized(original), + } + uploaded = [] + for name, (data, width, height) in generated.items(): + key = f"attachments/{attachment_id}/{name}" + await storage.put_object(key, data, content_types[name]) # network call: can fail transiently too + uploaded.append((name, key, data, width, height)) + except Exception: # noqa: BLE001 - covers S3 reachability (get+put) and image decode, not just decode + logger.exception("processing failed for attachment %s", attachment_id) + app_metrics.record_job("attachment.process.v1", "failed_attempt", time.monotonic() - started) + await _handle_failure(job, "processing_failed") + return + + async with db.pool().acquire() as conn: + async with conn.transaction(): + for name, key, data, width, height in uploaded: + digest = hashlib.sha256(data).hexdigest() + await conn.execute( + """ + INSERT INTO attachment_variants (attachment_id, name, content_type, size_bytes, width, height, sha256, s3_key) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (attachment_id, name) DO UPDATE SET + content_type=EXCLUDED.content_type, size_bytes=EXCLUDED.size_bytes, + width=EXCLUDED.width, height=EXCLUDED.height, sha256=EXCLUDED.sha256, s3_key=EXCLUDED.s3_key + """, + attachment_id, name, content_types[name], len(data), width, height, digest, key, + ) + await conn.execute( + "UPDATE attachments SET status='ready', error_code=NULL, updated_at=now() WHERE id=$1 AND status != 'ready'", + attachment_id, + ) + logger.info("attachment %s ready (attempt %d)", attachment_id, attempt) + app_metrics.record_job("attachment.process.v1", "success", time.monotonic() - started) + accepted_at = record["created_at"] + app_metrics.record_time_to_ready((dt.datetime.now(dt.timezone.utc) - accepted_at).total_seconds()) + + +async def _handle_failure(job: dict, error_code: str) -> None: + attachment_id = uuid.UUID(job["attachment_id"]) + job_id = uuid.UUID(job["job_id"]) + attempt = job["attempt"] + async with db.pool().acquire() as conn: + if attempt >= MAX_ATTEMPTS: + async with conn.transaction(): + await conn.execute( + "UPDATE attachments SET status='failed', error_code=$2, updated_at=now() WHERE id=$1", + attachment_id, error_code, + ) + await conn.execute( + "INSERT INTO job_failures (attachment_id, job_id, attempt, error_code) VALUES ($1,$2,$3,$4) " + "ON CONFLICT (attachment_id, job_id) DO NOTHING", + attachment_id, job_id, attempt, error_code, + ) + logger.warning("attachment %s permanently failed after %d attempts: %s", attachment_id, attempt, error_code) + return + # Transient: schedule a durable delayed retry instead of sleeping in-process. + delay = min(30, 2 ** (attempt - 1)) + random.uniform(0, 1) + await conn.execute( + """ + INSERT INTO job_retry_schedule (attachment_id, job_id, attempt, next_attempt_at) + VALUES ($1, $2, $3, now() + ($4 * interval '1 second')) + ON CONFLICT (attachment_id, job_id) DO UPDATE SET + attempt=EXCLUDED.attempt, next_attempt_at=EXCLUDED.next_attempt_at + """, + attachment_id, job_id, attempt + 1, delay, + ) + await conn.execute( + "UPDATE attachments SET status='queued', updated_at=now() WHERE id=$1 AND status != 'ready'", + attachment_id, + ) + logger.info("attachment %s scheduled for retry #%d in %.1fs", attachment_id, attempt + 1, delay) + + +async def retry_scheduler_loop() -> None: + while True: + try: + async with db.pool().acquire() as conn: + due = await conn.fetch( + "SELECT attachment_id, job_id, attempt FROM job_retry_schedule WHERE next_attempt_at <= now() LIMIT 50" + ) + for row in due: + job = { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": 1, + "attempt": row["attempt"], + "created_at": "1970-01-01T00:00:00Z", # informational only; not used for ordering + } + await broker.publish(job) + await conn.execute( + "DELETE FROM job_retry_schedule WHERE attachment_id=$1 AND job_id=$2", + row["attachment_id"], row["job_id"], + ) + except Exception: # noqa: BLE001 - scheduler must survive a transient hiccup + logger.exception("retry scheduler tick failed") + await asyncio.sleep(RETRY_SCAN_INTERVAL_SECONDS) + + +async def reclaim_loop() -> None: + """Hand this worker any job another (likely dead) worker left unacked + past JOB_TIMEOUT_SECONDS -- lease recovery for a crashed worker.""" + while True: + try: + for message_id, job in await broker.reclaim_stale(CONSUMER_NAME, JOB_TIMEOUT_SECONDS * 1000): + logger.info("reclaimed stale job %s (message %s)", job["job_id"], message_id) + await process_job(job) + await broker.ack(message_id) + except Exception: # noqa: BLE001 + logger.exception("reclaim tick failed") + await asyncio.sleep(JOB_TIMEOUT_SECONDS / 2) + + +async def consume_loop() -> None: + while True: + try: + for message_id, job in await broker.read(CONSUMER_NAME, count=1, block_ms=2000): + await process_job(job) + await broker.ack(message_id) + except Exception: # noqa: BLE001 - keep the loop alive across transient errors + logger.exception("consume tick failed") + await asyncio.sleep(1) + + +async def main() -> None: + global _tracer + telemetry.init(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker")) + app_metrics.init() + _tracer = trace.get_tracer("messenger-worker") + + database_url = read_secret("DATABASE_URL_FILE", "DATABASE_URL") + cache_url = read_secret("CACHE_URL_FILE", "CACHE_URL") + await db.init_pool(database_url) + broker.init(cache_url) + await broker.ensure_group() + storage.init( + endpoint=os.environ["S3_ENDPOINT"], + region=os.environ.get("S3_REGION", "us-east-1"), + access_key=read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"), + secret_key=read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"), + bucket=os.environ.get("S3_BUCKET", "messenger-attachments"), + ) + + loop = asyncio.get_event_loop() + + def _queue_sync(): + future = asyncio.run_coroutine_threadsafe(broker.depth_and_oldest_age_seconds(), loop) + return future.result(timeout=5) + + app_metrics.observable_gauges( + queue_depth_getter=_queue_sync, + worker_slots_getter=lambda: (_active_jobs, WORKER_CAPACITY), + ) + + logger.info("worker %s starting", CONSUMER_NAME) + await asyncio.gather(consume_loop(), reclaim_loop(), retry_scheduler_loop()) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..12727e8 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,263 @@ +name: mtusi2026lab6 + +services: + lb: + image: nginx:1.27.2 + ports: + - "8080:8080" + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + + api1: &api + build: . + image: mtusi2026lab6-app:local + environment: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: lab + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: http://seaweedfs:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + DEPLOYMENT_ENVIRONMENT: lab + INSTANCE_ID: api1 + secrets: &api-secrets + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + seaweedfs: + condition: service_healthy + otel-collector: + condition: service_started + healthcheck: &api-healthcheck + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 320M + + api2: + <<: *api + environment: + <<: *api-env + INSTANCE_ID: api2 + + worker: + build: . + image: mtusi2026lab6-app:local + command: ["python3", "-m", "app.worker"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: http://seaweedfs:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + DEPLOYMENT_ENVIRONMENT: lab + INSTANCE_ID: worker1 + secrets: *api-secrets + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + seaweedfs: + condition: service_healthy + otel-collector: + condition: service_started + deploy: + resources: + limits: + cpus: "1.0" + memory: 384M + + postgres: + image: postgres:16.4 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + redis: + image: redis:7.4.1 + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: + - redisdata:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + seaweedfs: + image: chrislusf/seaweedfs:3.73 + command: + - server + - -dir=/data + - -s3 + - -s3.port=8333 + - -s3.config=/etc/seaweedfs/s3.json + volumes: + - seaweed_data:/data + - ./secrets/s3.json:/etc/seaweedfs/s3.json:ro + healthcheck: + test: ["CMD", "nc", "-z", "127.0.0.1", "8333"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.114.0 + command: ["--config=/etc/otel-collector/config.yaml"] + volumes: + - ./infra/otel-collector/config.yaml:/etc/otel-collector/config.yaml:ro + depends_on: + - prometheus + - loki + - tempo + deploy: + resources: + limits: + cpus: "0.5" + memory: 200M + + prometheus: + image: prom/prometheus:v3.0.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + loki: + image: grafana/loki:3.2.1 + command: ["-config.file=/etc/loki/loki-config.yaml"] + volumes: + - ./infra/loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro + - loki_data:/loki + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + tempo: + image: grafana/tempo:2.6.1 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + grafana: + image: grafana/grafana:11.3.1 + ports: + - "3000:3000" + environment: + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_FEATURE_TOGGLES_ENABLE: traceqlEditor + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/etc/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + depends_on: + - prometheus + - loki + - tempo + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + +volumes: + pgdata: + redisdata: + seaweed_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + +secrets: + database_url: + file: ./secrets/database_url.txt + cache_url: + file: ./secrets/cache_url.txt + postgres_password: + file: ./secrets/postgres_password.txt + s3_access_key_id: + file: ./secrets/s3_access_key_id.txt + s3_secret_access_key: + file: ./secrets/s3_secret_access_key.txt diff --git a/evidence/incident1_worker_down.txt b/evidence/incident1_worker_down.txt new file mode 100644 index 0000000..bcf2e66 --- /dev/null +++ b/evidence/incident1_worker_down.txt @@ -0,0 +1,86 @@ +worker stopped, 3 uploads queued +After ~waiting with worker down, queue_backlog oldest_age_seconds exceeded alert threshold (120s): +queue_backlog{deployment_environment_name="lab",instance="api1",job="messenger-api",metric="depth",queue="attachment_jobs",service_instance_id="api1",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 10 +queue_backlog{deployment_environment_name="lab",instance="api1",job="messenger-api",metric="oldest_age_seconds",queue="attachment_jobs",service_instance_id="api1",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 1091.1752433776855 +queue_backlog{deployment_environment_name="lab",instance="api2",job="messenger-api",metric="depth",queue="attachment_jobs",service_instance_id="api2",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 10 +queue_backlog{deployment_environment_name="lab",instance="api2",job="messenger-api",metric="oldest_age_seconds",queue="attachment_jobs",service_instance_id="api2",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 1090.9032080173492 +queue_backlog{deployment_environment_name="lab",instance="worker1",job="messenger-worker",metric="depth",queue="attachment_jobs",service_instance_id="worker1",service_name="messenger-worker",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 7 +queue_backlog{deployment_environment_name="lab",instance="worker1",job="messenger-worker",metric="oldest_age_seconds",queue="attachment_jobs",service_instance_id="worker1",service_name="messenger-worker",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 1071.3482460975647 +--- Grafana alert rule state --- +[ + { + "annotations": { + "__orgId__": "1", + "description": "SLI: oldest_age_seconds from queue_backlog. SLO: < 120s under normal load. Runbook: check `worker_concurrency_slots` (busy==capacity sustained => worker saturated) and whether the worker container is running (`docker compose ps worker`); check messenger-worker logs in Loki for repeated processing_failed entries (S3/DB dependency down).\n", + "summary": "Oldest pending attachment job older than 2 minutes" + }, + "endsAt": "2026-10-29T22:51:00.000Z", + "fingerprint": "1e290b8bdee8bacd", + "receivers": [ + { + "name": "grafana-default-email" + } + ], + "startsAt": "2026-10-29T22:39:00.000Z", + "status": { + "inhibitedBy": [], + "silencedBy": [], + "state": "active" + }, + "updatedAt": "2026-10-29T22:49:00.041Z", + "generatorURL": "http://localhost:3000/alerting/grafana/messenger-growing-queue-lag/view", + "labels": { + "__alert_rule_uid__": "messenger-growing-queue-lag", + "alertname": "DatasourceNoData", + "datasource_uid": "Prometheus", + "grafana_folder": "Messenger", + "ref_id": "A", + "rulename": "Growing attachment queue lag", + "severity": "warning" + } + }, + { + "annotations": { + "__orgId__": "1", + "description": "SLI: 5xx / all business requests (excluding /health*), 5m window. SLO: error ratio < 5%. Runbook: check `db_pool_connections` (DB saturation?) and recent error-level entries in {service_name=\"messenger-api\"} in Loki; correlate via trace_id with Tempo to find the failing dependency.\n", + "summary": "5xx ratio on business requests above 5% for 2 minutes" + }, + "endsAt": "2026-10-29T22:51:30.000Z", + "fingerprint": "8e849c2fbff0680b", + "receivers": [ + { + "name": "grafana-default-email" + } + ], + "startsAt": "2026-10-29T22:39:00.000Z", + "status": { + "inhibitedBy": [], + "silencedBy": [], + "state": "active" + }, + "updatedAt": "2026-10-29T22:49:35.120Z", + "generatorURL": "http://localhost:3000/alerting/grafana/messenger-high-error-rate/view", + "labels": { + "__alert_rule_uid__": "messenger-high-error-rate", + "alertname": "DatasourceNoData", + "datasource_uid": "Prometheus", + "grafana_folder": "Messenger", + "ref_id": "A", + "rulename": "High HTTP error rate", + "severity": "critical" + } + } +] +--- after restarting worker --- +queue_backlog{deployment_environment_name="lab",instance="api1",job="messenger-api",metric="depth",queue="attachment_jobs",service_instance_id="api1",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 7 +queue_backlog{deployment_environment_name="lab",instance="api1",job="messenger-api",metric="oldest_age_seconds",queue="attachment_jobs",service_instance_id="api1",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 1121.2472867965698 +queue_backlog{deployment_environment_name="lab",instance="api2",job="messenger-api",metric="depth",queue="attachment_jobs",service_instance_id="api2",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 7 +queue_backlog{deployment_environment_name="lab",instance="api2",job="messenger-api",metric="oldest_age_seconds",queue="attachment_jobs",service_instance_id="api2",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 1125.980663061142 +queue_backlog{deployment_environment_name="lab",instance="worker1",job="messenger-worker",metric="depth",queue="attachment_jobs",service_instance_id="worker1",service_name="messenger-worker",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 7 +queue_backlog{deployment_environment_name="lab",instance="worker1",job="messenger-worker",metric="oldest_age_seconds",queue="attachment_jobs",service_instance_id="worker1",service_name="messenger-worker",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 1123.25155711174 +queue_backlog{deployment_environment_name="lab",instance="api1",job="messenger-api",metric="depth",queue="attachment_jobs",service_instance_id="api1",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 7 +queue_backlog{deployment_environment_name="lab",instance="api1",job="messenger-api",metric="oldest_age_seconds",queue="attachment_jobs",service_instance_id="api1",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 1141.295405626297 +queue_backlog{deployment_environment_name="lab",instance="api2",job="messenger-api",metric="depth",queue="attachment_jobs",service_instance_id="api2",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 7 +queue_backlog{deployment_environment_name="lab",instance="api2",job="messenger-api",metric="oldest_age_seconds",queue="attachment_jobs",service_instance_id="api2",service_name="messenger-api",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 1146.0297138690948 +queue_backlog{deployment_environment_name="lab",instance="worker1",job="messenger-worker",metric="depth",queue="attachment_jobs",service_instance_id="worker1",service_name="messenger-worker",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 7 +queue_backlog{deployment_environment_name="lab",instance="worker1",job="messenger-worker",metric="oldest_age_seconds",queue="attachment_jobs",service_instance_id="worker1",service_name="messenger-worker",service_version="0.1.0",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.29.0"} 1143.273363828659 diff --git a/evidence/incident2_db_down.txt b/evidence/incident2_db_down.txt new file mode 100644 index 0000000..ef50703 --- /dev/null +++ b/evidence/incident2_db_down.txt @@ -0,0 +1,20 @@ +=== Incident 2: PostgreSQL unavailable === +baseline (DB up): +http_code=200 time=0.005956s +--- postgres stopped --- +http_code=500 time=0.013984s +request body: {"error":{"code":"internal_error","message":"internal error","request_id":"8e7690fe8fcf47108c547d49a2dd108f"}} +X-Request-Id: 8e7690fe8fcf47108c547d49a2dd108f + +trace_id (from Loki log correlation): 0f8359335fabe9574a08406208226a5f +Span breakdown from Tempo: + GET /api/v1/users/me duration=10.7ms status=ERROR "gaierror: Temporary failure in name resolution" + http GET (outer middleware span) duration=11.7ms status=ok + +Diagnosis: the FAILING span has ERROR status and a DB-connection-specific +exception message, and total request duration stayed ~11ms -- this is a fast, +explicit dependency failure, NOT a slow handler (a slow handler would show a +long-duration span with NO error status, e.g. an expensive query actually +completing after several seconds). The combination of (a) 500 + ERROR span +status + (b) short duration is how this scenario is distinguished: a +"slow handler" incident would instead show long duration with status=ok. diff --git a/evidence/incident3_telemetry_down.txt b/evidence/incident3_telemetry_down.txt new file mode 100644 index 0000000..2537118 --- /dev/null +++ b/evidence/incident3_telemetry_down.txt @@ -0,0 +1,16 @@ +=== Incident 3: telemetry backend (Prometheus+Loki+Tempo) down === +--- API still serving business traffic with telemetry backends down --- +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +--- collector memory after backend outage (should stay bounded, not grow unbounded) --- +mtusi2026lab6-otel-collector-1 94.42MiB / 200MiB + +Collector logs during outage: repeated "Exporting failed. Will retry the +request after interval." for both otlphttp/tempo and otlphttp/loki exporters, +with increasing backoff intervals (4.1s, 6.4s, 7.5s, 9.8s...) -- default +retry_sender exponential backoff. Collector process stayed up throughout +(not restarted), memory stayed at 94MiB/200MiB limit (well under the +memory_limiter's 150MiB internal threshold), so buffered-but-unsent spans/ +logs/metrics did not grow memory without bound. Prometheus scraping also +simply failed silently (its own container was down) with no effect on the +Collector or the API. diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..208e593 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,91 @@ +{ + "title": "Messenger overview", + "uid": "messenger-overview", + "timezone": "utc", + "schemaVersion": 39, + "version": 1, + "time": {"from": "now-1h", "to": "now"}, + "refresh": "10s", + "panels": [ + { + "id": 1, "title": "HTTP requests/s by status class", "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "datasource": {"type": "prometheus", "uid": "Prometheus"}, + "targets": [{"expr": "sum(rate(http_requests_total{route!~\"/health.*\"}[1m])) by (status_class)", "legendFormat": "{{status_class}}"}], + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []} + }, + { + "id": 2, "title": "HTTP error ratio (5xx / all business requests)", "type": "stat", + "gridPos": {"h": 8, "w": 6, "x": 12, "y": 0}, + "datasource": {"type": "prometheus", "uid": "Prometheus"}, + "targets": [{"expr": "sum(rate(http_requests_total{status_class=\"5xx\", route!~\"/health.*\"}[5m])) / sum(rate(http_requests_total{route!~\"/health.*\"}[5m]))"}], + "fieldConfig": {"defaults": {"unit": "percentunit", "decimals": 2}, "overrides": []} + }, + { + "id": 3, "title": "HTTP duration p50/p95/p99", "type": "timeseries", + "gridPos": {"h": 8, "w": 6, "x": 18, "y": 0}, + "datasource": {"type": "prometheus", "uid": "Prometheus"}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{route!~\"/health.*\"}[5m])) by (le))", "legendFormat": "p50"}, + {"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{route!~\"/health.*\"}[5m])) by (le))", "legendFormat": "p95"}, + {"expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{route!~\"/health.*\"}[5m])) by (le))", "legendFormat": "p99"} + ], + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []} + }, + { + "id": 4, "title": "Jobs processed/s by outcome", "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 8}, + "datasource": {"type": "prometheus", "uid": "Prometheus"}, + "targets": [{"expr": "sum(rate(jobs_processed_total[1m])) by (outcome)", "legendFormat": "{{outcome}}"}], + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []} + }, + { + "id": 5, "title": "Job processing duration p95", "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 8}, + "datasource": {"type": "prometheus", "uid": "Prometheus"}, + "targets": [{"expr": "histogram_quantile(0.95, sum(rate(job_processing_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95 processing"}], + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []} + }, + { + "id": 6, "title": "Time to ready (upload -> ready) p95", "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 8}, + "datasource": {"type": "prometheus", "uid": "Prometheus"}, + "targets": [{"expr": "histogram_quantile(0.95, sum(rate(time_to_ready_seconds_bucket[5m])) by (le))", "legendFormat": "p95 time-to-ready"}], + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []} + }, + { + "id": 7, "title": "Queue depth (pending jobs)", "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 16}, + "datasource": {"type": "prometheus", "uid": "Prometheus"}, + "targets": [{"expr": "queue_backlog{metric=\"depth\"}", "legendFormat": "pending"}], + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []} + }, + { + "id": 8, "title": "Oldest pending job age", "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 16}, + "datasource": {"type": "prometheus", "uid": "Prometheus"}, + "targets": [{"expr": "queue_backlog{metric=\"oldest_age_seconds\"}", "legendFormat": "oldest age"}], + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []} + }, + { + "id": 9, "title": "DB pool used/idle", "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 16}, + "datasource": {"type": "prometheus", "uid": "Prometheus"}, + "targets": [{"expr": "db_pool_connections", "legendFormat": "{{service_instance_id}} {{state}}"}], + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []} + }, + { + "id": 10, "title": "Worker busy/capacity", "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 24}, + "datasource": {"type": "prometheus", "uid": "Prometheus"}, + "targets": [{"expr": "worker_concurrency_slots", "legendFormat": "{{state}}"}], + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []} + }, + { + "id": 11, "title": "Recent API logs", "type": "logs", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 24}, + "datasource": {"type": "loki", "uid": "Loki"}, + "targets": [{"expr": "{service_name=\"messenger-api\"}"}] + } + ] +} diff --git a/infra/grafana/provisioning/alerting/rules.yaml b/infra/grafana/provisioning/alerting/rules.yaml new file mode 100644 index 0000000..611b495 --- /dev/null +++ b/infra/grafana/provisioning/alerting/rules.yaml @@ -0,0 +1,68 @@ +apiVersion: 1 + +groups: + - orgId: 1 + name: messenger-slo + folder: Messenger + interval: 30s + rules: + - uid: messenger-high-error-rate + title: High HTTP error rate + condition: C + for: 2m + data: + - refId: A + relativeTimeRange: {from: 300, to: 0} + datasourceUid: Prometheus + model: + expr: sum(rate(http_requests_total{status_class="5xx", route!~"/health.*"}[5m])) / sum(rate(http_requests_total{route!~"/health.*"}[5m])) + instant: true + refId: A + - refId: C + datasourceUid: "__expr__" + model: + type: threshold + expression: A + conditions: + - evaluator: {type: gt, params: [0.05]} + refId: C + labels: + severity: critical + annotations: + summary: "5xx ratio on business requests above 5% for 2 minutes" + description: > + SLI: 5xx / all business requests (excluding /health*), 5m window. + SLO: error ratio < 5%. Runbook: check `db_pool_connections` (DB + saturation?) and recent error-level entries in {service_name="messenger-api"} + in Loki; correlate via trace_id with Tempo to find the failing dependency. + + - uid: messenger-growing-queue-lag + title: Growing attachment queue lag + condition: C + for: 2m + data: + - refId: A + relativeTimeRange: {from: 60, to: 0} + datasourceUid: Prometheus + model: + expr: max(queue_backlog{metric="oldest_age_seconds"}) + instant: true + refId: A + - refId: C + datasourceUid: "__expr__" + model: + type: threshold + expression: A + conditions: + - evaluator: {type: gt, params: [120]} + refId: C + labels: + severity: warning + annotations: + summary: "Oldest pending attachment job older than 2 minutes" + description: > + SLI: oldest_age_seconds from queue_backlog. SLO: < 120s under + normal load. Runbook: check `worker_concurrency_slots` (busy==capacity + sustained => worker saturated) and whether the worker container is + running (`docker compose ps worker`); check messenger-worker logs + in Loki for repeated processing_failed entries (S3/DB dependency down). diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..252d3cf --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /etc/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..37433d2 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,29 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: Prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 5s + + - name: Loki + type: loki + uid: Loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherRegex: "trace_id\":\"(\\w+)\"" + url: "$${__value.raw}" + datasourceUid: tempo + + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + uid: tempo diff --git a/infra/loki/loki-config.yaml b/infra/loki/loki-config.yaml new file mode 100644 index 0000000..2b68434 --- /dev/null +++ b/infra/loki/loki-config.yaml @@ -0,0 +1,29 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 0000000..952e0db --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,32 @@ +worker_processes 1; +# A single worker keeps the round-robin counter shared and deterministic for +# the lab's small replica count; a production LB would instead rely on a +# shared/consistent balancing policy across workers. +events { worker_connections 1024; } + +http { + upstream messenger_api { + # Passive health check: a backend failing 1 probe is removed for 3s, + # which keeps failover detection well under the 10s budget. + server api1:8080 max_fails=1 fail_timeout=3s; + server api2:8080 max_fails=1 fail_timeout=3s; + } + + server { + listen 8080; + # Slightly above the app's own 10 MiB attachment limit so the API -- + # not Nginx's default HTML error page -- produces the contract's + # JSON 413 envelope. The API still enforces the exact byte budget. + client_max_body_size 11M; + + location / { + proxy_pass http://messenger_api; + proxy_connect_timeout 1s; + proxy_read_timeout 5s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + } +} diff --git a/infra/otel-collector/config.yaml b/infra/otel-collector/config.yaml new file mode 100644 index 0000000..3850138 --- /dev/null +++ b/infra/otel-collector/config.yaml @@ -0,0 +1,47 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + # Bounds the Collector's own memory so an unavailable backend cannot grow + # RAM without limit; it sheds/refuses data under pressure instead. + memory_limiter: + check_interval: 1s + limit_mib: 150 + spike_limit_mib: 40 + batch: + send_batch_size: 512 + timeout: 5s + +exporters: + otlphttp/tempo: + endpoint: http://tempo:4318 + tls: + insecure: true + otlphttp/loki: + endpoint: http://loki:3100/otlp + tls: + insecure: true + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..f73d1f7 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,8 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8889"] diff --git a/infra/seaweedfs/s3.json.template b/infra/seaweedfs/s3.json.template new file mode 100644 index 0000000..4f0ab30 --- /dev/null +++ b/infra/seaweedfs/s3.json.template @@ -0,0 +1,18 @@ +{ + "identities": [ + { + "name": "messenger-api", + "credentials": [ + { + "accessKey": "__S3_ACCESS_KEY_ID__", + "secretKey": "__S3_SECRET_ACCESS_KEY__" + } + ], + "actions": [ + "Read:messenger-attachments", + "Write:messenger-attachments", + "List:messenger-attachments" + ] + } + ] +} diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..5740d27 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,24 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + grpc: + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..005e1b4 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,33 @@ +-- Initial schema: users, chats, membership, messages. +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Keyset pagination over (chat_id, created_at, id) avoids a full scan per page. +CREATE INDEX IF NOT EXISTS messages_chat_keyset_idx ON messages (chat_id, created_at, id); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 0000000..f2a124f --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,15 @@ +-- Idempotency store for POST message: scope = (user, method, path, key). +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL, + method TEXT NOT NULL, + path TEXT NOT NULL, + idem_key TEXT NOT NULL, + request_hash TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + response_status INT, + response_body JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, idem_key) +); + +CREATE INDEX IF NOT EXISTS idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 0000000..7635855 --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,25 @@ +-- Attachment metadata lives in Postgres; bytes live in private S3. s3_key is +-- server-generated and never derived from the user-supplied filename. +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'ready', + error_code TEXT, + s3_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments (chat_id); + +-- Many-to-many: a message can reference up to 10 attachments of its own chat. +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id), + PRIMARY KEY (message_id, attachment_id) +); diff --git a/migrations/0004_jobs.sql b/migrations/0004_jobs.sql new file mode 100644 index 0000000..aa5a057 --- /dev/null +++ b/migrations/0004_jobs.sql @@ -0,0 +1,53 @@ +-- Variant outputs (thumbnail/crop/optimized) produced by the worker. +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + name TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + width INT NOT NULL, + height INT NOT NULL, + sha256 TEXT NOT NULL, + s3_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, name) +); + +-- Transactional outbox: written in the SAME transaction as the attachments +-- INSERT, so a crash between COMMIT and broker publish leaves a recoverable +-- row instead of a lost job. A relay (in-process background task) publishes +-- unpublished rows and marks them, so publish is retried, not re-created. +CREATE TABLE IF NOT EXISTS job_outbox ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + job_id UUID NOT NULL, + pipeline_version INT NOT NULL DEFAULT 1, + attempt INT NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + published_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS job_outbox_unpublished_idx ON job_outbox (created_at) WHERE published_at IS NULL; + +-- Durable delayed-retry schedule (Redis Streams has no native delay): a +-- failed attempt is parked here with a next_attempt_at instead of sleeping +-- in-process, so a worker crash during the backoff window does not lose the +-- retry -- any worker's scheduler tick will still publish it on time. +CREATE TABLE IF NOT EXISTS job_retry_schedule ( + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + job_id UUID NOT NULL, + attempt INT NOT NULL, + next_attempt_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (attachment_id, job_id) +); + +CREATE INDEX IF NOT EXISTS job_retry_schedule_due_idx ON job_retry_schedule (next_attempt_at); + +-- Explicit failure record for DLQ-style operator visibility and replay. +CREATE TABLE IF NOT EXISTS job_failures ( + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + job_id UUID NOT NULL, + attempt INT NOT NULL, + error_code TEXT NOT NULL, + failed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, job_id) +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..547b916 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +fastapi==0.115.6 +uvicorn[standard]==0.32.1 +asyncpg==0.30.0 +redis==5.2.1 +argon2-cffi==23.1.0 +pydantic==2.10.3 +boto3==1.35.74 +Pillow==11.0.0 +opentelemetry-api==1.29.0 +opentelemetry-sdk==1.29.0 +opentelemetry-exporter-otlp-proto-http==1.29.0 +opentelemetry-instrumentation-fastapi==0.50b0 +opentelemetry-instrumentation-asyncpg==0.50b0 +opentelemetry-instrumentation-redis==0.50b0 diff --git a/scripts/reconcile_attachments.py b/scripts/reconcile_attachments.py new file mode 100644 index 0000000..67b9b14 --- /dev/null +++ b/scripts/reconcile_attachments.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Find S3 objects with no matching `attachments` row (e.g. PUT succeeded but +the PostgreSQL INSERT that followed it never committed -- a crash between the +two writes). Dry-run by default; --delete removes orphans older than the +grace period, never anything referenced in the database. + +Usage: + python3 scripts/reconcile_attachments.py --grace-minutes 60 + python3 scripts/reconcile_attachments.py --grace-minutes 60 --delete +""" +import argparse +import asyncio +import datetime as dt +import os +import sys + +import asyncpg +import boto3 + + +def read_secret(file_env, inline_env): + path = os.environ.get(file_env) + if path and os.path.isfile(path): + return open(path, encoding="utf-8").read().strip() + return os.environ[inline_env] + + +async def main(): + p = argparse.ArgumentParser() + p.add_argument("--grace-minutes", type=int, default=60, + help="Only objects older than this are candidates (protects in-flight uploads).") + p.add_argument("--delete", action="store_true", help="Actually delete orphans (default: dry-run).") + args = p.parse_args() + + database_url = read_secret("DATABASE_URL_FILE", "DATABASE_URL") + conn = await asyncpg.connect(database_url) + known_keys = {r["s3_key"] for r in await conn.fetch("SELECT s3_key FROM attachments")} + await conn.close() + + s3 = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + region_name=os.environ.get("S3_REGION", "us-east-1"), + aws_access_key_id=read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"), + aws_secret_access_key=read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"), + ) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=args.grace_minutes) + + orphans = [] + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket, Prefix="attachments/"): + for obj in page.get("Contents", []): + if obj["Key"] in known_keys: + continue + if obj["LastModified"] > cutoff: + continue # might still be an in-flight upload; not an orphan yet + orphans.append(obj["Key"]) + + print(f"{len(orphans)} orphaned object(s) older than {args.grace_minutes} minutes with no DB row.") + for key in orphans: + print(" ", key) + if args.delete: + s3.delete_object(Bucket=bucket, Key=key) + if orphans and args.delete: + print(f"Deleted {len(orphans)} orphan(s).") + elif orphans: + print("Dry-run: nothing deleted. Re-run with --delete to remove them.") + + +if __name__ == "__main__": + sys.exit(asyncio.run(main()) or 0) diff --git a/scripts/replay_failed.py b/scripts/replay_failed.py new file mode 100755 index 0000000..d6b8608 --- /dev/null +++ b/scripts/replay_failed.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Explicit operator replay for a failed attachment: resets status to +'queued' and republishes a fresh job (new job_id, attempt=1). Never replays +an attachment that is already 'ready' -- the same protection the worker +itself applies against stale duplicates. + +Usage: python3 scripts/replay_failed.py +""" +import asyncio +import datetime as dt +import os +import sys +import uuid + +import asyncpg + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from app import broker # noqa: E402 + + +def read_secret(file_env, inline_env): + path = os.environ.get(file_env) + if path and os.path.isfile(path): + return open(path, encoding="utf-8").read().strip() + return os.environ[inline_env] + + +async def main(): + if len(sys.argv) != 2: + print("usage: replay_failed.py ", file=sys.stderr) + return 2 + attachment_id = uuid.UUID(sys.argv[1]) + + conn = await asyncpg.connect(read_secret("DATABASE_URL_FILE", "DATABASE_URL")) + record = await conn.fetchrow("SELECT status FROM attachments WHERE id=$1", attachment_id) + if record is None: + print("no such attachment", file=sys.stderr) + return 1 + if record["status"] == "ready": + print("attachment is already ready; refusing to replay a finished job") + return 0 + if record["status"] != "failed": + print(f"attachment status is '{record['status']}', not 'failed'; nothing to replay") + return 0 + + job_id = uuid.uuid4() + await conn.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=$1", attachment_id) + await conn.close() + + broker.init(read_secret("CACHE_URL_FILE", "CACHE_URL")) + job = { + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": 1, + "attempt": 1, + "created_at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"), + } + await broker.publish(job) + print(f"replayed attachment {attachment_id} as job {job_id}") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..b320b2d --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +wait_healthy() { + for svc in "$@"; do + for _ in $(seq 1 60); do + status=$(docker compose ps --format '{{.Health}}' "$svc" 2>/dev/null || true) + [ "$status" = "healthy" ] && break + sleep 1 + done + done +} + +docker compose restart postgres redis seaweedfs +wait_healthy postgres redis seaweedfs + +docker compose restart otel-collector prometheus loki tempo grafana + +docker compose restart api1 api2 +wait_healthy api1 api2 + +docker compose restart worker +docker compose restart lb +exit 0 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..5bbb18e --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +# Stops exactly the API replica identified by TARGET_INSTANCE_ID (api1 or api2), +# leaving it stopped. The other replica keeps serving traffic through the LB. +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID must be set by the test harness}" +case "$TARGET_INSTANCE_ID" in + api1|api2) ;; + *) echo "unknown instance id: $TARGET_INSTANCE_ID" >&2; exit 1 ;; +esac +docker compose stop "$TARGET_INSTANCE_ID" +exit 0 -- 2.54.0