diff --git a/.env.example b/.env.example index 2b32655..c663dba 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,37 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Local synthetic credentials for compose. Copy to .env; never commit .env. +# The application reads DATABASE_URL, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY and BROKER_URL. +# *_FILE variables from the course worksheet are accepted too when the direct variable is empty. + +POSTGRES_USER=messenger +POSTGRES_PASSWORD=replace-with-local-secret +POSTGRES_DB=messenger +S3_ACCESS_KEY=replace-with-local-secret +S3_SECRET_KEY=replace-with-local-secret +RABBITMQ_USER=messenger +RABBITMQ_PASSWORD=replace-with-local-secret +GRAFANA_ADMIN_PASSWORD=replace-with-local-secret + APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: +DATABASE_URL=postgresql://messenger:replace-with-local-secret@postgres:5432/messenger SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=replace-with-local-secret +S3_SECRET_ACCESS_KEY=replace-with-local-secret MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +BROKER_URL=amqp://messenger:replace-with-local-secret@rabbitmq:5672/%2F WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..3fea2fc 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 6 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Казанцева Виктория / БПИ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 2, SLO и обнаружение инцидентов -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: мессенджер лабораторий 1–5 (сессии, keyset, идемпотентность, вложения, две API-реплики, PostgreSQL, SeaweedFS, RabbitMQ, worker) и наблюдаемость: OTLP в Collector, Prometheus, Loki, Tempo, Grafana provisioning, burn-rate алерты. +Команды запуска из чистого clone: см. REPORT.md. Кратко: `python scripts/render-s3-config.py`, затем `docker compose --env-file .env up -d --build`. +Результаты проверок и ссылка на REPORT.md: smoke и check_contract PASS; persistence и failover PASS; доказательства в REPORT.md и `evidence/`. +Ограничения / известные ошибки: на Windows `.sh` хуки не исполняются напрямую; прогон шёл через Git Bash. Короткий и длинный эксперименты SLO шли подряд, поэтому детекция длинной деградации быстрее холодного старта. +Источники: документация OpenTelemetry, Prometheus, Grafana. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] Проверки `scripts/check_contract.py` и `tests/smoke.py` (то же, что make check и make test) прошли; на Windows они запускались через `PYTHONUTF8=1 python`. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.gitignore b/.gitignore index b5a7189..dd177a6 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,8 @@ coverage/ .DS_Store .idea/ .vscode/ +faults/* +!faults/README.md +infra/seaweed/s3.json # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..16b48bf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.12.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + AWS_EC2_METADATA_DISABLED=true + +WORKDIR /app + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app/app + +USER app +EXPOSE 8080 8081 + +CMD ["python", "-m", "app.main"] diff --git a/REPORT.md b/REPORT.md index b97ae42..33fc0cd 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,121 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Казанцева Виктория / БПИ2301 +- Целевая оценка: 5, трек 2 (SLO и обнаружение инцидентов) +- Commit/PR: не публиковался +- Источники: OpenTelemetry Collector, Prometheus histograms, W3C Trace Context, Grafana provisioning. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Compose v2 и Python 3.9+ на хосте (проверки). Образы закреплены в `compose.yaml`, приложение собирается из этого репозитория. -## Архитектура и решения +``` +python scripts/render-s3-config.py +docker compose --env-file .env config --quiet +docker compose --env-file .env build api-1 +docker compose --env-file .env up -d +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`scripts/render-s3-config.py` читает `S3_ACCESS_KEY` и `S3_SECRET_KEY` из локального `.env` и пишет gitignored `infra/seaweed/s3.json`. Шаблон переменных — `.env.example`. Файл `.env` не коммитится. + +Схема БД применяется дважды и одинаково: `app/schema.sql` монтируется в `docker-entrypoint-initdb.d` и повторно накатывается приложением под `pg_advisory_lock`. Бакет S3 создаёт API/worker при старте, если его ещё нет. Очередь `attachment.process.v1` объявляет приложение. + +Готовность: `GET http://127.0.0.1:8080/health/ready` возвращает 200. На Windows публичные скрипты запускать с `PYTHONUTF8=1`, потому что `Path.read_text()` в проверках курса читает JSON в кодировке локали. + +``` +python scripts/check_contract.py +python tests/smoke.py --base-url http://127.0.0.1:8080 +``` + +Остановка без удаления данных: `docker compose --env-file .env stop`. Удаление учебных томов только явно: `docker compose --env-file .env down -v`. + +`scripts/restart.sh` перезапускает postgres, SeaweedFS, RabbitMQ, обе API-реплики и worker, ждёт их healthcheck и затем перечитывает upstream в nginx. Тома не удаляются. `scripts/stop-one.sh` останавливает только `api-1` или `api-2` из `TARGET_INSTANCE_ID`. На Windows эти `.sh` напрямую не исполняются (`WinError 193`); проверка ниже запускала их через Git Bash, не меняя `tests/resilience.py`. + +## Архитектура + +Бизнес-API лабораторий 1–5 не менялся. Две реплики `api-1` и `api-2` за nginx (`127.0.0.1:8080`, round-robin). PostgreSQL — источник истины для пользователей, сессий, чатов, сообщений, идемпотентности и outbox. SeaweedFS хранит байты вложений (`staging/`, `originals/`, `variants/`). RabbitMQ — durable очередь `attachment.process.v1`, persistent messages, confirm и ручной ack. Worker отдельно от API. + +Запись вложения: проверка доступа, байты в staging, строка attachment и outbox в одной транзакции, затем publish. Сбой брокера после commit не отменяет `202 queued`: poller допубликует outbox. Worker идемпотентен, lease 60 с, повтор того же `traceparent`. + +Наблюдаемость: API и worker шлют OTLP/HTTP protobuf на Collector (`:4318`). Collector: memory limiter 192 MiB, удаление URL/SQL/Authorization, batch, очередь exporter 200 и retry не дольше 30 с. Дальше Prometheus (scrape `:8889`), Loki (OTLP), Tempo (gRPC). Grafana только запрашивает эти три хранилища и не хранит сигналы сама. Дашборд `messenger-overview` и datasources поднимаются provisioning-ом из Git. Правила — `infra/prometheus/rules.yml`. + +| Роль | Сервис | Образ | +| --- | --- | --- | +| API, worker | api-1, api-2, worker | локальная сборка `mtusi-lab6-messenger:1.6.0` на `python:3.12.8-slim-bookworm` | +| Балансировщик | nginx | `nginx:1.27.4-alpine` | +| БД | postgres | `postgres:16.6-alpine` | +| S3 | s3 | `chrislusf/seaweedfs:4.51` | +| Брокер | rabbitmq | `rabbitmq:3.13.7-management-alpine` | +| Collector | otel-collector | `otel/opentelemetry-collector-contrib:0.120.0` | +| Метрики | prometheus | `prom/prometheus:v2.55.1` | +| Логи | loki | `grafana/loki:3.3.2` | +| Трассы | tempo | `grafana/tempo:2.7.2` | +| UI | grafana | `grafana/grafana:11.5.2` | + +Prometheus хранит ряды и состояние алертов. Loki хранит структурные JSON-логи. Tempo хранит спаны. Grafana их показывает. + +Метрики не используют `user_id`, `chat_id`, `message_id`, `attachment_id` как labels. Маршрут в метрике — шаблон. В логах есть `trace_id` и `span_id`. Пароли, токены, текст сообщений и байты картинок в telemetry не пишутся. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда | Наблюдение | Файл | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Контракт skeleton | `PYTHONUTF8=1 python scripts/check_contract.py` | PASS | вывод в этой сессии | +| Публичный smoke | `PYTHONUTF8=1 python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS после включённой telemetry | — | +| Готовность не зависит от Grafana | `GET /health/ready` при остановленном Tempo | 200, бизнес-GET тоже 200 | `evidence/incidents.json` | +| Метрики API и worker | PromQL после smoke | есть `http_server_requests_total`, `http_server_duration_seconds_bucket`, `messenger_jobs_total`, `messenger_job_duration_seconds_bucket`, глубина/возраст очереди, пул БД, busy/capacity/consumers | `scripts/probe_signals.py` | +| Лог и трасса одного upload | Loki `job.ready` и Tempo по тому же `trace_id` | лог `attempt: 2` и трасса ниже | `evidence/retry-trace.json` | +| Дашборд из Git | `GET /api/dashboards/uid/messenger-overview` | uid `messenger-overview`, Grafana 11.5.2 | provisioning в `infra/grafana` | +| p50/p95/p99 из histogram | панели дашборда, `histogram_quantile` по сумме bucket | не среднее и не среднее p99 реплик | `infra/grafana/dashboards/messenger.json` | +| W3C API → broker → worker, retry, S3/DB | один upload с `process_fail_remaining=1` | см. цепочку | `evidence/retry-trace.json` | +| SLI/SLO и burn rate | правила Prometheus | семь alert rules загружены | `infra/prometheus/rules.yml` | +| Короткий всплеск не открывает fast burn | `scripts/slo_experiment.py` | pending, не firing | `evidence/slo-track2.json` | +| Длинная деградация | тот же прогон | fast burn firing через 51.6 с, resolved через 70.6 с | `evidence/slo-track2.json` | +| Остановка worker | `scripts/incident_followup.py` | consumers firing 75.5 с, queue lag 105.7 с, оба resolved за 20.1 с | `evidence/incidents.json` | +| Остановка Tempo | Tempo stop 40 с | 76× HTTP 200, failed spans 9→24, очередь 22, RAM collector 68.8 MiB / 320 MiB | `evidence/incidents.json` | +| Сохранность | публичный persistence через `scripts/restart.sh` | PASS, сессия и вложение на месте | — | +| Отказ одной реплики | публичный failover через `scripts/stop-one.sh` | 2 instance id до отказа, 77 успешных записей, 0 ошибок, восстановление 2.094 с | — | +| Нагрузка | `python tests/load.py --duration 15 --concurrency 2` | 1247 успехов, 0 ошибок, 82.7 rps, p50 31 мс, p95/p99 47 мс | `evidence/load.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Цепочка одного upload, trace `cf5025ad9d9a2a248da8335a0c6caee2`: -## Эксперименты и отказы +1. `POST /api/v1/chats/{chat_id}/attachments` — корень. +2. `attachment.process publish`, parent = POST. +3. Два `attachment.process consume` с одним и тем же parent = publish. Первый — внедрённый transient (`job.retry`, attempt 1). Второй — sibling, attempt 2, `job.ready`. +4. У второго consume дети `s3.get`, два `s3.put` и `db.transaction`. У POST свои `db.transaction` и `s3.put` staging. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Перцентили на дашборде считаются как `histogram_quantile` от `sum by (le)` rate bucket. p99 реплик не усредняется: сначала складываются bucket, потом берётся квантиль. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +## SLI / SLO -## Ограничения и следующие шаги +SLI API: доля ответов бизнес-маршрутов, которые не 5xx. Знаменатель — все бизнес-запросы. `/health/*` и несовпавший маршрут исключены. 4xx не ошибка SLI. SLO: 99% за скользящие окна burn rate (бюджет 1%). -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Быстрое сжигание: и 1 мин, и 5 мин строго выше 14.4% дольше 30 с, при трафике больше 0.2 rps. +- Медленное: и 5 мин, и 15 мин выше 6% дольше 1 мин. +- Дополнительно: error rate 5 мин выше 5% дольше 1 мин; возраст старейшей job больше 30 с дольше 1 мин; число consumers по данным API равно 0 дольше 1 мин; p95 времени до ready выше 10 с. + +Доставка наружу не настраивалась. Состояние — локальные ряды `ALERTS`. + +## Трек 2 + +Гипотеза: при уже идущем успешном трафике всплеск около 10 с поднимает минутное окно и не поднимает пятиминутное выше 14.4%, поэтому fast burn не становится firing. Непрерывные 5xx дольше примерно 45 с поднимают оба коротких окна, и fast burn зажигается. Остановленный worker виден по возрасту очереди и нулю consumers, а не по error rate API. Остановленный Tempo не мешает принять сообщения, exporter упирается в очередь и лимит памяти. + +Методика на этом хосте (16 ГБ RAM, лимиты контейнеров в `compose.yaml`): закрытый цикл `GET /api/v1/chats` около 2 rps. Прогрев 90 с дал 180 ответов 200 и ноль alerts. Затем 10 с с `http_error_ratio=1`: 20 ответов 500. Сразу после этого `ApiHighErrorRate` и `ApiErrorBudgetSlowBurn` были только pending, `ApiErrorBudgetFastBurn` не появился. Дальше те же 100% ошибок: fast burn, high error rate и slow burn стали firing через 51.6 с (104 ответа 500). После снятия fault fast burn ушёл через 70.6 с; high и slow оставались firing и к началу следующего прогона (около двух минут спустя) уже были inactive. + +Ограничения измерения. Короткий всплеск и длинный прогон шли подряд, поэтому ошибки всплеска ещё лежали в пятиминутном окне и ускорили детект длинной деградации относительно «холодного» старта. p99 нагрузки 15 с на 2 потоках совпал с p95 (47 мс): выборка короткая, это закрытый цикл, не открытый поток. Время resolved для slow burn снято по факту «уже нет» к 21:49:48 UTC, без секундомера. `AttachmentReadyLatencyHigh` стал firing из-за job, которая ждала остановленный worker: это следствие того же инцидента, гистограмма 15 мин ещё держит медленное завершение. + +## Два инцидента + +Остановка worker. Сразу после stop API показывал `messenger_worker_consumers = 0` и растущий возраст очереди. `WorkerConsumersZero` firing на 75.5 с, `QueueLagHigh` на 105.7 с при возрасте около 98 с. После start оба alert исчезли за 20.1 с. Runbook: `docs/runbooks/queue-lag.md`. + +Остановка Tempo на 40 с. Бизнес-GET: 76 ответов 200, ошибок нет. `otelcol_exporter_send_failed_spans` вырос с 9 до 24. Размер очереди exporter дошёл до 22 при лимите 200. Память collector 68.8 MiB при лимите контейнера 320 MiB и memory limiter 192 MiB. Сообщения не терялись. Runbook: `docs/runbooks/telemetry-backend.md`. + +Как отличить медленную БД от медленного handler: `db_slow_ms` спит внутри спана `db.transaction`, `handler_slow_ms` спит в серверном спане и вне `db.transaction`. + +## Ограничения + +Публичные `tests/resilience.py` на этом Windows запускают `.sh` как Win32 и получают `WinError 193`. Сами хуки прогнаны тем же тестом через Git Bash. На Linux `make test-persistence` и `make test-failover` вызывают скрипты напрямую. + +Чтобы вообще скачать образы, с диска C: были удалены неиспользуемые образы других локальных проектов. Их тома не удалялись. Образы этих проектов при следующем запуске нужно скачать снова. + +SeaweedFS выбран вместо MinIO: образы `minio/minio` с Docker Hub в этой среде не скачиваются. Контракт разрешает SeaweedFS. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..fdeefa7 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger application for lab 6.""" diff --git a/app/asgi.py b/app/asgi.py new file mode 100644 index 0000000..1315c9d --- /dev/null +++ b/app/asgi.py @@ -0,0 +1,124 @@ +import asyncio +import random +import time +import uuid + +from opentelemetry.trace import SpanKind + +from app import faults, http, telemetry +from app.errors import AppError +from app.settings import settings + + +class _TooLarge(Exception): + pass + + +async def _read_body(receive, limit): + chunks = [] + size = 0 + while True: + message = await receive() + if message["type"] == "http.disconnect": + return None + if message["type"] != "http.request": + continue + data = message.get("body", b"") or b"" + size += len(data) + if size > limit: + raise _TooLarge() + chunks.append(data) + if not message.get("more_body"): + return b"".join(chunks) + + +def _too_large(scope, limit): + for name, value in scope.get("headers", []): + if name == b"content-length": + try: + return int(value) > limit + except ValueError: + return True + return False + + +async def app(scope, receive, send): + if scope["type"] == "lifespan": + while True: + message = await receive() + if message["type"] == "lifespan.startup": + telemetry.log_event("service.listening", port=8080) + await send({"type": "lifespan.startup.complete"}) + elif message["type"] == "lifespan.shutdown": + await send({"type": "lifespan.shutdown.complete"}) + return + return + if scope["type"] != "http": + return + method = scope.get("method", "GET") + path = scope.get("path", "/") + raw_headers = {} + for name, value in scope.get("headers", []): + raw_headers[name.decode("latin-1").lower()] = value.decode("latin-1") + request_id = http.request_id_from(raw_headers) + template, _name, _groups = http.match(method, path) + limit = http.body_limit(path) + started = time.perf_counter() + status = 500 + response_headers = {} + body = b"" + with telemetry.tracer().start_as_current_span( + f"{method} {template}", + kind=SpanKind.SERVER, + attributes={"http.request.method": method, "http.route": template}, + ) as span: + try: + if _too_large(scope, limit): + raise AppError(413, "payload_too_large", "Слишком большой объект") + try: + payload = await _read_body(receive, limit) + except _TooLarge as exc: + raise AppError(413, "payload_too_large", "Слишком большой объект") from exc + if payload is None: + return + if method in {"POST", "PUT"} and path.rstrip("/").endswith("/attachments") and len(payload) > settings.max_upload: + raise AppError(413, "payload_too_large", "Слишком большой объект") + query = http.query_from(scope.get("query_string", b"").decode("ascii", "replace")) + if path.startswith("/api/") and random.random() < faults.http_error_ratio(): + raise AppError(500, "internal_error", "Внутренняя ошибка") + + def run(): + return http.handle(method, path, query, raw_headers, payload) + + template, status, response_headers, body = await asyncio.to_thread(run) + span.set_attribute("http.route", template) + except Exception as exc: + status, response_headers, body = http.error_response(exc, request_id) + if not isinstance(exc, AppError): + telemetry.log_event("internal_error", severity="ERROR", error_type=type(exc).__name__, request_id=request_id) + if status >= 500: + telemetry.mark_error(span) + span.set_attribute("error.type", type(exc).__name__) + duration = time.perf_counter() - started + span.set_attribute("http.response.status_code", status) + telemetry.record_http(method, template, status, duration) + telemetry.log_event( + "http.request.completed", + request_id=request_id, + http_route=template, + http_method=method, + status=status, + duration_ms=int(duration * 1000), + ) + header_list = [ + (b"x-request-id", request_id.encode("ascii")), + (b"x-instance-id", settings.instance_id.encode("ascii")), + ] + for key, value in response_headers.items(): + header_list.append((key.lower().encode("latin-1"), value.encode("latin-1"))) + await send({"type": "http.response.start", "status": status, "headers": header_list}) + await send({"type": "http.response.body", "body": body}) + + +def new_request_id(): + return uuid.uuid4().hex diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 0000000..b9517fe --- /dev/null +++ b/app/broker.py @@ -0,0 +1,153 @@ +import base64 +import json +import uuid +from urllib.parse import quote, urlparse, unquote +from urllib.request import Request, urlopen + +import pika + +from app import telemetry +from app.db import jsonb +from app.settings import settings + +QUEUE = "attachment.process.v1" + + +def _params(): + parsed = urlparse(settings.broker_url) + return pika.ConnectionParameters( + host=parsed.hostname or "rabbitmq", + port=parsed.port or 5672, + virtual_host=unquote(parsed.path[1:] or "/"), + credentials=pika.PlainCredentials(unquote(parsed.username or ""), unquote(parsed.password or "")), + heartbeat=60, + blocked_connection_timeout=2, + socket_timeout=2, + connection_attempts=1, + retry_delay=0, + ) + + +def _credentials(): + parsed = urlparse(settings.broker_url) + return unquote(parsed.username or ""), unquote(parsed.password or ""), unquote(parsed.path[1:] or "/") + + +def init(): + connection = pika.BlockingConnection(_params()) + try: + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True) + finally: + connection.close() + + +def ready(): + try: + connection = pika.BlockingConnection(_params()) + try: + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True, passive=True) + finally: + connection.close() + return True + except Exception: + return False + + +def insert_outbox(conn, payload, created_at): + conn.execute( + """ + INSERT INTO outbox (job_id, attachment_id, payload, created_at) + VALUES (%s, %s, %s, %s) + """, + (uuid.UUID(str(payload["job_id"])), uuid.UUID(str(payload["attachment_id"])), jsonb(payload), created_at), + ) + + +def publish_pending(conn, job_id=None): + query = """ + SELECT job_id, payload + FROM outbox + WHERE published_at IS NULL + """ + args = [] + if job_id is not None: + query += " AND job_id = %s" + args.append(job_id) + query += " ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 20" + rows = conn.execute(query, args).fetchall() + for row in rows: + payload = row["payload"] + if isinstance(payload, str): + payload = json.loads(payload) + _publish(payload) + conn.execute("UPDATE outbox SET published_at = NOW() WHERE job_id = %s", (row["job_id"],)) + return len(rows) + + +def _publish(payload): + carrier = {} + if payload.get("traceparent"): + carrier["traceparent"] = payload["traceparent"] + if payload.get("tracestate"): + carrier["tracestate"] = payload["tracestate"] + parent = telemetry.extract_context(carrier) + with telemetry.tracer().start_as_current_span( + "attachment.process publish", + context=parent, + kind=telemetry.trace.SpanKind.PRODUCER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": QUEUE, + "messaging.operation.type": "publish", + }, + ): + outgoing = telemetry.inject_carrier() + body = dict(payload) + if outgoing.get("traceparent"): + body["traceparent"] = outgoing["traceparent"] + if outgoing.get("tracestate"): + body["tracestate"] = outgoing["tracestate"] + _send(body, outgoing) + telemetry.log_event( + "job.published", + job_id=str(payload.get("job_id")), + attempt=int(payload.get("attempt") or 1), + ) + + +def _send(payload, headers): + connection = pika.BlockingConnection(_params()) + try: + channel = connection.channel() + channel.confirm_delivery() + channel.basic_publish( + exchange="", + routing_key=QUEUE, + body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + headers=headers, + ), + mandatory=True, + ) + finally: + connection.close() + + +def consumer_count(): + user, password, vhost = _credentials() + url = ( + settings.rabbit_management_url.rstrip("/") + + "/api/queues/" + + quote(vhost, safe="") + + "/" + + quote(QUEUE, safe="") + ) + token = base64.b64encode(f"{user}:{password}".encode("utf-8")).decode("ascii") + request = Request(url, headers={"Authorization": "Basic " + token}) + with urlopen(request, timeout=2) as response: + data = json.loads(response.read().decode("utf-8")) + return int(data.get("consumers") or 0) diff --git a/app/codec.py b/app/codec.py new file mode 100644 index 0000000..22f86b4 --- /dev/null +++ b/app/codec.py @@ -0,0 +1,139 @@ +import base64 +import datetime as dt +import hashlib +import json +import re +import uuid + +from app.errors import AppError, bad_request + +USERNAME = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def utcnow(): + return dt.datetime.now(dt.timezone.utc) + + +def iso(value): + if value.tzinfo is None: + value = value.replace(tzinfo=dt.timezone.utc) + value = value.astimezone(dt.timezone.utc) + return value.strftime("%Y-%m-%dT%H:%M:%S.") + f"{value.microsecond:06d}Z" + + +def parse_iso(value): + if not isinstance(value, str) or not value.endswith("Z"): + raise bad_request() + try: + return dt.datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as exc: + raise bad_request() from exc + + +def parse_uuid(value): + if not isinstance(value, str): + raise bad_request() + try: + return str(uuid.UUID(value)) + except ValueError as exc: + raise bad_request() from exc + + +def require_text(value, minimum, maximum, allow_blank=False): + if not isinstance(value, str): + raise bad_request() + if "\x00" in value or any(ord(ch) < 32 or ord(ch) == 127 for ch in value): + raise bad_request() + if not allow_blank and not value.strip(): + raise bad_request() + if not (minimum <= len(value) <= maximum): + raise bad_request() + return value + + +def canonical_message(text, attachment_ids): + body = {"attachment_ids": list(attachment_ids), "text": text} + raw = json.dumps(body, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + return hashlib.sha256(raw.encode("utf-8")).digest() + + +def encode_cursor(kind, resource_id, created_at, row_id): + raw = f"{kind}|{resource_id}|{iso(created_at)}|{row_id}" + return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=") + + +def decode_cursor(token, kind, resource_id): + if not isinstance(token, str) or not token: + raise bad_request() + padded = token + ("=" * (-len(token) % 4)) + try: + raw = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8") + cursor_kind, cursor_resource, stamp, row_id = raw.split("|", 3) + created_at = parse_iso(stamp) + row_uuid = str(uuid.UUID(row_id)) + except (ValueError, UnicodeError, AppError) as exc: + raise bad_request() from exc + if cursor_kind != kind or cursor_resource != resource_id: + raise bad_request() + return created_at, row_uuid + + +def fitted_size(width, height, box=256): + if width < 1 or height < 1: + raise ValueError("dimensions") + if width <= box and height <= box: + return width, height + if width >= height: + return box, max(1, (height * box) // width) + return max(1, (width * box) // height), box + + +def detect_image(data, declared): + if data.startswith(b"\x89PNG\r\n\x1a\n"): + detected = "image/png" + elif data.startswith(b"\xff\xd8\xff"): + detected = "image/jpeg" + else: + return None + if declared != detected: + return None + return detected + + +def media_type(header): + if not header or not isinstance(header, str): + return "" + return header.split(";", 1)[0].strip().lower() + + +def loads_object(raw): + try: + text = raw.decode("utf-8") + except UnicodeError as exc: + raise bad_request() from exc + + def pairs(items): + obj = {} + for key, value in items: + if key in obj: + raise ValueError("duplicate") + obj[key] = value + return obj + + try: + value = json.loads(text, object_pairs_hook=pairs) + except (ValueError, UnicodeError) as exc: + raise bad_request() from exc + if not isinstance(value, dict): + raise bad_request() + return value + + +def expect_fields(data, allowed, required): + if set(data) - set(allowed): + raise bad_request() + for key in required: + if key not in data: + raise bad_request() diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..a478aef --- /dev/null +++ b/app/db.py @@ -0,0 +1,98 @@ +import time +from contextlib import contextmanager +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool, PoolTimeout + +from app import telemetry +from app.errors import AppError, unavailable +from app.faults import db_fail, db_slow_seconds +from app.settings import settings + +pool = None + + +def init(): + global pool + pool = ConnectionPool( + settings.database_url, + min_size=1, + max_size=settings.pool_max, + timeout=2, + max_waiting=16, + check=ConnectionPool.check_connection, + kwargs={ + "row_factory": dict_row, + "autocommit": False, + "connect_timeout": 3, + "options": "-c timezone=UTC -c statement_timeout=4000", + }, + open=True, + reconnect_timeout=2, + ) + + +def migrate(): + script = Path(__file__).with_name("schema.sql").read_text(encoding="utf-8") + statements = [] + buffer = [] + for line in script.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("--"): + continue + buffer.append(line) + if stripped.endswith(";"): + statements.append("\n".join(buffer)) + buffer = [] + with pool.connection() as conn: + conn.execute("SELECT pg_advisory_lock(861006)") + for statement in statements: + conn.execute(statement) + conn.execute("SELECT pg_advisory_unlock(861006)") + conn.commit() + + +def jsonb(value): + return Jsonb(value) + + +@contextmanager +def connection(): + if db_fail(): + raise unavailable() + delay = db_slow_seconds() + started = time.perf_counter() + span = telemetry.tracer().start_as_current_span( + "db.transaction", + attributes={"db.system": "postgresql"}, + ) + span.__enter__() + try: + if delay: + time.sleep(delay) + with pool.connection() as conn: + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + except AppError: + raise + except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError) as exc: + raise unavailable() from exc + finally: + span.__exit__(None, None, None) + telemetry.record_db(time.perf_counter() - started) + + +def stats(): + if pool is None: + return 0, 0 + data = pool.get_stats() + used = int(data.get("pool_size", 0)) - int(data.get("pool_available", 0)) + waiting = int(data.get("requests_waiting", 0)) + return max(used, 0), waiting diff --git a/app/domain.py b/app/domain.py new file mode 100644 index 0000000..834cf82 --- /dev/null +++ b/app/domain.py @@ -0,0 +1,584 @@ +import datetime as dt +import hashlib +import uuid + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError +from argon2.low_level import Type +from psycopg.errors import UniqueViolation + +from app import broker, storage, telemetry +from app.codec import ( + USERNAME, + IDEMPOTENCY, + canonical_message, + decode_cursor, + detect_image, + encode_cursor, + expect_fields, + iso, + parse_uuid, + require_text, + utcnow, +) +from app.db import connection, jsonb +from app.errors import bad_request, conflict, not_found, too_large, unauthorized, unsupported +from app.images import validate_image +from app.settings import settings + +BASIC = 'Basic realm="messenger", charset="UTF-8"' +BEARER = 'Bearer realm="messenger"' +_hasher = PasswordHasher(time_cost=1, memory_cost=8192, parallelism=1, hash_len=32, salt_len=16, type=Type.ID) +_dummy = None + + +def init_hasher(): + global _dummy + _dummy = _hasher.hash("not-a-real-password") + + +def _user_json(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": iso(row["created_at"]), + } + + +def _token_hash(token): + return hashlib.sha256(token.encode("utf-8")).digest() + + +def _password(value): + if not isinstance(value, str) or "\x00" in value or not (12 <= len(value) <= 128): + raise bad_request() + return value + + +def as_uuid(value): + if isinstance(value, uuid.UUID): + return value + return uuid.UUID(str(value)) + + +def _username(value): + if not isinstance(value, str) or USERNAME.fullmatch(value) is None: + raise bad_request() + return value + + +def register(data): + expect_fields(data, ["username", "display_name", "password"], ["username", "display_name", "password"]) + username = _username(data["username"]) + display_name = require_text(data["display_name"], 1, 100) + password = _password(data["password"]) + password_hash = _hasher.hash(password) + user_id = uuid.uuid4() + created_at = utcnow() + try: + with connection() as conn: + conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (user_id, username, display_name, password_hash, created_at), + ) + row = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ).fetchone() + except UniqueViolation as exc: + raise conflict("Имя пользователя занято") from exc + return 201, _user_json(row) + + +def login(username, password): + if not isinstance(username, str) or not isinstance(password, str) or "\x00" in username or "\x00" in password: + raise unauthorized(BASIC) + with connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s", + (username,), + ).fetchone() + stored = row["password_hash"] if row else _dummy + try: + _hasher.verify(stored, password) + valid = row is not None + except (VerifyMismatchError, VerificationError, InvalidHashError): + valid = False + if not valid: + raise unauthorized(BASIC) + token = secrets_token() + expires_at = utcnow() + dt.timedelta(seconds=settings.session_ttl) + with connection() as conn: + conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (_token_hash(token), row["id"], expires_at), + ) + return 201, { + "token": token, + "token_type": "Bearer", + "expires_at": iso(expires_at), + "user": _user_json(row), + } + + +def secrets_token(): + import secrets + + return secrets.token_urlsafe(32) + + +def current_user(token): + if not token or len(token) < 32: + raise unauthorized(BEARER) + with connection() as conn: + row = conn.execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > NOW() + """, + (_token_hash(token),), + ).fetchone() + if not row: + raise unauthorized(BEARER) + return row + + +def ready_check(): + with connection() as conn: + conn.execute("SELECT 1") + if not storage.ready() or not broker.ready(): + raise unavailable() + + +def revoke(token): + user = current_user(token) + with connection() as conn: + conn.execute( + "UPDATE sessions SET revoked_at = NOW() WHERE token_hash = %s AND revoked_at IS NULL", + (_token_hash(token),), + ) + return user + + +def create_chat(user, data): + expect_fields(data, ["title", "member_ids"], ["title", "member_ids"]) + title = require_text(data["title"], 1, 100) + raw_members = data["member_ids"] + if not isinstance(raw_members, list) or not (1 <= len(raw_members) <= 100): + raise bad_request() + members = [parse_uuid(item) for item in raw_members] + if len(set(members)) != len(members): + raise bad_request() + if str(user["id"]) not in members: + raise bad_request() + chat_id = uuid.uuid4() + created_at = utcnow() + with connection() as conn: + found = conn.execute("SELECT id FROM users WHERE id = ANY(%s)", ([as_uuid(item) for item in members],)).fetchall() + if len(found) != len(members): + raise bad_request() + conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat_id, title, created_at), + ) + for position, member_id in enumerate(members): + conn.execute( + "INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)", + (chat_id, as_uuid(member_id), position), + ) + return 201, { + "id": str(chat_id), + "title": title, + "member_ids": members, + "created_at": iso(created_at), + } + + +def _chat_json(conn, row): + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position", + (row["id"],), + ).fetchall() + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(item["user_id"]) for item in members], + "created_at": iso(row["created_at"]), + } + + +def list_chats(user, limit, cursor): + created_at = None + row_id = None + if cursor is not None: + created_at, row_id = decode_cursor(cursor, "chats", str(user["id"])) + with connection() as conn: + if created_at is None: + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s + ORDER BY c.created_at, c.id + LIMIT %s + """, + (user["id"], limit + 1), + ).fetchall() + else: + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s AND (c.created_at, c.id) > (%s, %s) + ORDER BY c.created_at, c.id + LIMIT %s + """, + (user["id"], created_at, as_uuid(row_id), limit + 1), + ).fetchall() + extra = len(rows) > limit + rows = rows[:limit] + items = [_chat_json(conn, row) for row in rows] + next_cursor = None + if extra and rows: + last = rows[-1] + next_cursor = encode_cursor("chats", str(user["id"]), last["created_at"], str(last["id"])) + return {"items": items, "next_cursor": next_cursor} + + +def get_chat(user, chat_id): + with connection() as conn: + row = _visible_chat(conn, user["id"], chat_id) + return _chat_json(conn, row) + + +def _visible_chat(conn, user_id, chat_id): + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = %s AND m.user_id = %s + """, + (as_uuid(chat_id), as_uuid(user_id)), + ).fetchone() + if not row: + raise not_found() + return row + + +def _message_json(row, attachment_ids): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": iso(row["created_at"]), + "attachment_ids": attachment_ids, + } + + +def _attachment_ids(conn, message_id): + rows = conn.execute( + "SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position", + (message_id,), + ).fetchall() + return [str(row["attachment_id"]) for row in rows] + + +def create_message(user, chat_id, key, data, path): + chat_uuid = as_uuid(chat_id) + if IDEMPOTENCY.fullmatch(key or "") is None: + raise bad_request() + expect_fields(data, ["text", "attachment_ids"], ["text"]) + if not isinstance(data["text"], str) or "\x00" in data["text"] or len(data["text"]) > 4000: + raise bad_request() + text = data["text"] + if "attachment_ids" in data: + raw_ids = data["attachment_ids"] + if not isinstance(raw_ids, list) or len(raw_ids) > 10: + raise bad_request() + attachment_ids = [parse_uuid(item) for item in raw_ids] + if len(set(attachment_ids)) != len(attachment_ids): + raise bad_request() + else: + attachment_ids = [] + if not text.strip() and not attachment_ids: + raise bad_request() + body_hash = canonical_message(text, attachment_ids) + lock = int.from_bytes(hashlib.sha256(f"{user['id']}|POST|{path}|{key}".encode("utf-8")).digest()[:8], "big", signed=True) + with connection() as conn: + _visible_chat(conn, user["id"], chat_id) + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,)) + existing = conn.execute( + """ + SELECT body_hash, response_json, created_at + FROM idempotency + WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s + """, + (user["id"], path, key), + ).fetchone() + if existing and existing["created_at"] >= utcnow() - dt.timedelta(seconds=settings.idempotency_ttl): + if bytes(existing["body_hash"]) != body_hash: + raise conflict("Ключ идемпотентности уже использован с другим телом") + stored = existing["response_json"] + if isinstance(stored, str): + import json + + stored = json.loads(stored) + return 201, stored + if existing: + conn.execute( + "DELETE FROM idempotency WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s", + (user["id"], path, key), + ) + if attachment_ids: + rows = conn.execute( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s)", + ([as_uuid(item) for item in attachment_ids],), + ).fetchall() + found = {str(row["id"]): row for row in rows} + if len(found) != len(attachment_ids): + raise not_found() + for attachment_id in attachment_ids: + row = found[attachment_id] + if str(row["chat_id"]) != chat_id: + raise not_found() + if row["status"] == "failed": + raise conflict("Нельзя прикрепить неуспешное вложение") + message_id = uuid.uuid4() + created_at = utcnow() + conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", + (message_id, chat_uuid, user["id"], text, created_at), + ) + for position, attachment_id in enumerate(attachment_ids): + conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)", + (message_id, as_uuid(attachment_id), position), + ) + stored = conn.execute( + "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = %s", + (message_id,), + ).fetchone() + payload = _message_json(stored, attachment_ids) + conn.execute( + """ + INSERT INTO idempotency (user_id, method, path, idem_key, body_hash, status_code, response_json, created_at) + VALUES (%s, 'POST', %s, %s, %s, 201, %s, %s) + """, + (user["id"], path, key, body_hash, jsonb(payload), created_at), + ) + return 201, payload + + +def list_messages(user, chat_id, limit, cursor): + chat_uuid = as_uuid(chat_id) + created_at = None + row_id = None + if cursor is not None: + created_at, row_id = decode_cursor(cursor, "messages", chat_id) + with connection() as conn: + _visible_chat(conn, user["id"], chat_uuid) + if created_at is None: + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + ORDER BY created_at, id + LIMIT %s + """, + (chat_uuid, limit + 1), + ).fetchall() + else: + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s AND (created_at, id) > (%s, %s) + ORDER BY created_at, id + LIMIT %s + """, + (chat_uuid, created_at, as_uuid(row_id), limit + 1), + ).fetchall() + extra = len(rows) > limit + rows = rows[:limit] + items = [_message_json(row, _attachment_ids(conn, row["id"])) for row in rows] + next_cursor = None + if extra and rows: + last = rows[-1] + next_cursor = encode_cursor("messages", chat_id, last["created_at"], str(last["id"])) + return {"items": items, "next_cursor": next_cursor} + + +def _variants(conn, attachment_id): + rows = conn.execute( + """ + SELECT name, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = 1 + ORDER BY name + """, + (attachment_id,), + ).fetchall() + return [ + { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "width": row["width"], + "height": row["height"], + "sha256": row["sha256"], + } + for row in rows + ] + + +def _attachment_json(row, variants): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": variants, + "error_code": row["error_code"], + "created_at": iso(row["created_at"]), + "updated_at": iso(row["updated_at"]), + } + + +def upload(user, chat_id, filename, content_type, data): + import hashlib as hl + + if not data: + raise unsupported() + if len(data) > settings.max_upload: + raise too_large() + detected = detect_image(data, content_type) + if not detected: + raise unsupported() + validate_image(data) + with connection() as conn: + _visible_chat(conn, user["id"], chat_id) + attachment_id = uuid.uuid4() + created_at = utcnow() + digest = hl.sha256(data).hexdigest() + staging_key = f"staging/{attachment_id}" + storage.put(staging_key, data, detected, "staging") + carrier = telemetry.inject_carrier() + job_id = uuid.uuid4() + payload = { + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": 1, + "attempt": 1, + "created_at": iso(created_at), + } + if carrier.get("traceparent"): + payload["traceparent"] = carrier["traceparent"] + if carrier.get("tracestate"): + payload["tracestate"] = carrier["tracestate"] + row = { + "id": attachment_id, + "chat_id": chat_id, + "owner_id": user["id"], + "filename": filename, + "content_type": detected, + "size_bytes": len(data), + "sha256": digest, + "status": "queued", + "error_code": None, + "created_at": created_at, + "updated_at": created_at, + } + with connection() as conn: + _visible_chat(conn, user["id"], chat_id) + conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, + error_code, staging_key, pipeline_version, attempt, created_at, updated_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', NULL, %s, 1, 0, %s, %s) + """, + ( + attachment_id, + as_uuid(chat_id), + user["id"], + filename, + detected, + len(data), + digest, + staging_key, + created_at, + created_at, + ), + ) + broker.insert_outbox(conn, payload, created_at) + try: + with connection() as conn: + broker.publish_pending(conn, job_id) + except Exception: + telemetry.log_event("job.publish_deferred", severity="WARNING", job_id=str(job_id)) + telemetry.log_event("attachment.accepted", attachment_id=str(attachment_id), size_bytes=len(data), content_type=detected) + return 202, _attachment_json(row, []) + + +def get_attachment(user, attachment_id): + attachment_uuid = as_uuid(attachment_id) + with connection() as conn: + row = _visible_attachment(conn, user["id"], attachment_uuid) + return _attachment_json(row, _variants(conn, attachment_uuid)) + + +def _visible_attachment(conn, user_id, attachment_id): + row = conn.execute("SELECT * FROM attachments WHERE id = %s", (as_uuid(attachment_id),)).fetchone() + if not row: + raise not_found() + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (row["chat_id"], user_id), + ).fetchone() + if not member: + raise not_found() + return row + + +def download(user, attachment_id, variant): + attachment_uuid = as_uuid(attachment_id) + if variant not in {"original", "thumbnail", "crop", "optimized"}: + raise bad_request() + with connection() as conn: + row = _visible_attachment(conn, user["id"], attachment_uuid) + if row["status"] != "ready": + raise conflict("Вложение ещё не готово") + if variant == "original": + key = row["original_key"] + content_type = row["content_type"] + else: + found = conn.execute( + """ + SELECT object_key, content_type + FROM attachment_variants + WHERE attachment_id = %s AND name = %s AND pipeline_version = 1 + """, + (attachment_uuid, variant), + ).fetchone() + if not found: + raise not_found() + key = found["object_key"] + content_type = found["content_type"] + return storage.get(key, variant), content_type diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 0000000..2868d06 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,40 @@ +class AppError(Exception): + def __init__(self, status, code, message, headers=None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message="Неверный запрос"): + return AppError(400, "invalid_request", message) + + +def unauthorized(challenge): + return AppError( + 401, + "unauthorized", + "Требуется аутентификация", + {"WWW-Authenticate": challenge}, + ) + + +def not_found(): + return AppError(404, "not_found", "Объект не найден") + + +def conflict(message="Конфликт"): + return AppError(409, "conflict", message) + + +def too_large(message="Слишком большой объект"): + return AppError(413, "payload_too_large", message) + + +def unsupported(message="Неподдерживаемый тип изображения"): + return AppError(415, "unsupported_media_type", message) + + +def unavailable(message="Сервис временно недоступен"): + return AppError(503, "unavailable", message) diff --git a/app/faults.py b/app/faults.py new file mode 100644 index 0000000..5e91539 --- /dev/null +++ b/app/faults.py @@ -0,0 +1,73 @@ +from pathlib import Path + +from app.settings import settings + + +def _text(name): + path = Path(settings.fault_dir) / name + try: + return path.read_text(encoding="utf-8").strip() + except (FileNotFoundError, OSError): + return "" + + +def db_slow_seconds(): + raw = _text("db_slow_ms") + if not raw: + return 0.0 + try: + value = float(raw) + except ValueError: + return 0.0 + if value <= 0: + return 0.0 + return min(value, 10000.0) / 1000.0 + + +def db_fail(): + return _text("db_fail") == "1" + + +def http_error_ratio(): + raw = _text("http_error_ratio") + if not raw: + return 0.0 + try: + value = float(raw) + except ValueError: + return 0.0 + if value <= 0: + return 0.0 + return min(value, 1.0) + + +def handler_slow_seconds(): + raw = _text("handler_slow_ms") + if not raw: + return 0.0 + try: + value = float(raw) + except ValueError: + return 0.0 + if value <= 0: + return 0.0 + return min(value, 10000.0) / 1000.0 + + +def consume_process_failure(): + path = Path(settings.fault_dir) / "process_fail_remaining" + try: + raw = path.read_text(encoding="utf-8").strip() + except (FileNotFoundError, OSError): + return False + try: + remaining = int(raw or "0") + except ValueError: + return False + if remaining <= 0: + return False + try: + path.write_text(str(remaining - 1), encoding="utf-8") + except OSError: + return False + return True diff --git a/app/http.py b/app/http.py new file mode 100644 index 0000000..be9c015 --- /dev/null +++ b/app/http.py @@ -0,0 +1,221 @@ +import json +import re +from urllib.parse import parse_qs + +from app import domain, faults +from app.codec import REQUEST_ID, loads_object, media_type, require_text +from app.errors import AppError, bad_request, not_found, unauthorized +from app.settings import settings + +_ROUTES = [ + ("GET", re.compile(r"^/health/live$"), "/health/live", "live"), + ("GET", re.compile(r"^/health/ready$"), "/health/ready", "ready"), + ("POST", re.compile(r"^/api/v1/users$"), "/api/v1/users", "register"), + ("GET", re.compile(r"^/api/v1/users/me$"), "/api/v1/users/me", "me"), + ("POST", re.compile(r"^/api/v1/auth/sessions$"), "/api/v1/auth/sessions", "login"), + ("DELETE", re.compile(r"^/api/v1/auth/sessions/current$"), "/api/v1/auth/sessions/current", "logout"), + ("POST", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "create_chat"), + ("GET", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "list_chats"), + ("GET", re.compile(r"^/api/v1/chats/([^/]+)$"), "/api/v1/chats/{chat_id}", "get_chat"), + ("POST", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "create_message"), + ("GET", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "list_messages"), + ("POST", re.compile(r"^/api/v1/chats/([^/]+)/attachments$"), "/api/v1/chats/{chat_id}/attachments", "upload"), + ("GET", re.compile(r"^/api/v1/attachments/([^/]+)$"), "/api/v1/attachments/{attachment_id}", "get_attachment"), + ("GET", re.compile(r"^/api/v1/attachments/([^/]+)/content$"), "/api/v1/attachments/{attachment_id}/content", "download"), +] + + +def match(method, path): + for route_method, pattern, template, name in _ROUTES: + found = pattern.fullmatch(path) + if found and route_method == method: + return template, name, found.groups() + return "unmatched", "unmatched", () + + +def _single(query, name): + values = query.get(name) + if not values: + return None + if len(values) != 1 or values[0] == "": + raise bad_request() + return values[0] + + +def _limit(query): + raw = _single(query, "limit") + if raw is None: + return 50 + if re.fullmatch(r"[1-9][0-9]*", raw) is None: + raise bad_request() + value = int(raw) + if value > 100: + raise bad_request() + return value + + +def _json_body(headers, body): + if media_type(headers.get("content-type", "")) != "application/json": + raise bad_request() + if not body: + raise bad_request() + return loads_object(body) + + +def _bearer(headers): + value = headers.get("authorization", "") + if not value.lower().startswith("bearer "): + raise unauthorized(domain.BEARER) + token = value.split(" ", 1)[1].strip() + return token + + +def _basic(headers): + import base64 + + value = headers.get("authorization", "") + if not value.lower().startswith("basic "): + raise unauthorized(domain.BASIC) + try: + decoded = base64.b64decode(value.split(" ", 1)[1].strip(), validate=True).decode("utf-8") + except (ValueError, UnicodeError) as exc: + raise unauthorized(domain.BASIC) from exc + if ":" not in decoded: + raise unauthorized(domain.BASIC) + return decoded.split(":", 1) + + +def _filename(query): + raw = _single(query, "filename") + if raw is None: + raise bad_request() + if len(raw) < 1 or len(raw) > 255 or not raw.strip(): + raise bad_request() + if any(ord(ch) < 32 or ord(ch) == 127 or ch in "/\\" for ch in raw): + raise bad_request() + return raw + + +def handle(method, path, query, headers, body): + template, name, groups = match(method, path) + if name == "unmatched": + raise not_found() + if name not in {"live", "ready"}: + delay = faults.handler_slow_seconds() + if delay: + import time + + time.sleep(delay) + if name == "live": + return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"}) + if name == "ready": + domain.ready_check() + return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"}) + if name == "register": + status, payload = domain.register(_json_body(headers, body)) + return template, status, _json_headers(), _dump(payload) + if name == "login": + if body: + raise bad_request() + username, password = _basic(headers) + status, payload = domain.login(username, password) + return template, status, _json_headers(), _dump(payload) + if name == "me": + user = domain.current_user(_bearer(headers)) + return template, 200, _json_headers(), _dump(domain._user_json(user)) + if name == "logout": + domain.revoke(_bearer(headers)) + return template, 204, {}, b"" + if name == "create_chat": + user = domain.current_user(_bearer(headers)) + status, payload = domain.create_chat(user, _json_body(headers, body)) + return template, status, _json_headers(), _dump(payload) + if name == "list_chats": + user = domain.current_user(_bearer(headers)) + payload = domain.list_chats(user, _limit(query), _single(query, "cursor")) + return template, 200, _json_headers(), _dump(payload) + if name == "get_chat": + user = domain.current_user(_bearer(headers)) + payload = domain.get_chat(user, _uuid_segment(groups[0])) + return template, 200, _json_headers(), _dump(payload) + if name == "create_message": + user = domain.current_user(_bearer(headers)) + chat_id = _uuid_segment(groups[0]) + status, payload = domain.create_message( + user, + chat_id, + headers.get("idempotency-key"), + _json_body(headers, body), + path, + ) + return template, status, _json_headers(), _dump(payload) + if name == "list_messages": + user = domain.current_user(_bearer(headers)) + payload = domain.list_messages(user, _uuid_segment(groups[0]), _limit(query), _single(query, "cursor")) + return template, 200, _json_headers(), _dump(payload) + if name == "upload": + user = domain.current_user(_bearer(headers)) + content_type = media_type(headers.get("content-type", "")) + status, payload = domain.upload(user, _uuid_segment(groups[0]), _filename(query), content_type, body) + return template, status, _json_headers(), _dump(payload) + if name == "get_attachment": + user = domain.current_user(_bearer(headers)) + payload = domain.get_attachment(user, _uuid_segment(groups[0])) + return template, 200, _json_headers(), _dump(payload) + if name == "download": + user = domain.current_user(_bearer(headers)) + variant = _single(query, "variant") or "original" + data, content_type = domain.download(user, _uuid_segment(groups[0]), variant) + return template, 200, {"Content-Type": content_type}, data + raise not_found() + + +def _uuid_segment(value): + from app.codec import parse_uuid + + return parse_uuid(value) + + +def _json_headers(): + return {"Content-Type": "application/json; charset=utf-8"} + + +def _dump(payload): + return json.dumps(payload, ensure_ascii=False).encode("utf-8") + + +def error_response(exc, request_id): + if isinstance(exc, AppError): + status = exc.status + code = exc.code + message = exc.message + headers = dict(exc.headers) + else: + status = 500 + code = "internal_error" + message = "Внутренняя ошибка" + headers = {} + headers["Content-Type"] = "application/json; charset=utf-8" + body = _dump({"error": {"code": code, "message": message, "request_id": request_id}}) + return status, headers, body + + +def request_id_from(headers): + incoming = headers.get("x-request-id", "") + if REQUEST_ID.fullmatch(incoming): + return incoming + import uuid + + return uuid.uuid4().hex + + +def query_from(raw): + if not raw: + return {} + return parse_qs(raw, keep_blank_values=True, strict_parsing=False) + + +def body_limit(path): + if "/attachments" in path and path.rstrip("/").endswith("attachments"): + return settings.max_upload + 1 + return 1_000_000 diff --git a/app/images.py b/app/images.py new file mode 100644 index 0000000..b4e940a --- /dev/null +++ b/app/images.py @@ -0,0 +1,62 @@ +import io + +from PIL import Image, ImageOps, UnidentifiedImageError + +from app.codec import fitted_size +from app.errors import AppError, too_large, unsupported +from app.settings import settings + +Image.MAX_IMAGE_PIXELS = settings.max_pixels + + +def _open(data): + try: + with Image.open(io.BytesIO(data)) as image: + image.verify() + with Image.open(io.BytesIO(data)) as image: + width, height = image.size + if width < 1 or height < 1 or width * height > settings.max_pixels: + raise too_large() + image.load() + return image.copy() + except AppError: + raise + except Image.DecompressionBombError as exc: + raise too_large() from exc + except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombWarning) as exc: + raise unsupported() from exc + + +def validate_image(data): + _open(data).close() + + +def _rgb(image): + if image.mode == "RGB": + return image + if image.mode in ("RGBA", "LA") or (image.mode == "P" and "transparency" in image.info): + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + return image.convert("RGB") + + +def thumbnail_jpeg(data): + image = _open(data) + try: + image = ImageOps.exif_transpose(image) + image = _rgb(image) + target = fitted_size(image.width, image.height, 256) + if (image.width, image.height) != target: + image = image.resize(target, Image.Resampling.LANCZOS) + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=85, optimize=True) + blob = buffer.getvalue() + return blob, image.width, image.height + except AppError: + raise + except (UnidentifiedImageError, OSError, ValueError) as exc: + raise unsupported() from exc + finally: + image.close() diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..e85f78a --- /dev/null +++ b/app/main.py @@ -0,0 +1,50 @@ +import os +import time + +import uvicorn + +from app import broker, db, domain, storage, telemetry +from app.asgi import app +from app.poller import start + + +def _retry(label, action): + last = None + for _ in range(40): + try: + action() + return + except Exception as exc: + last = exc + telemetry.log_event("dependency.retry", severity="WARNING", dependency=label, error_type=type(exc).__name__) + time.sleep(1) + raise last + + +def main(): + telemetry.init() + _retry("postgres", db.init) + db.migrate() + _retry("s3", storage.init) + _retry("broker", broker.init) + domain.init_hasher() + start() + uvicorn.run( + app, + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + access_log=False, + log_config={ + "version": 1, + "disable_existing_loggers": False, + "loggers": { + "uvicorn": {"level": "WARNING"}, + "uvicorn.error": {"level": "WARNING"}, + "uvicorn.access": {"level": "WARNING"}, + }, + }, + ) + + +if __name__ == "__main__": + main() diff --git a/app/poller.py b/app/poller.py new file mode 100644 index 0000000..f0895bf --- /dev/null +++ b/app/poller.py @@ -0,0 +1,44 @@ +import threading + +from app import broker, db, telemetry + + +def _once(): + try: + with db.connection() as conn: + row = conn.execute( + """ + SELECT COUNT(*) AS depth, + EXTRACT(EPOCH FROM (NOW() - MIN(created_at))) AS age + FROM attachments + WHERE status IN ('queued', 'processing') + """ + ).fetchone() + depth = int(row["depth"] or 0) + age = float(row["age"] or 0) if depth else 0.0 + used, waiting = db.stats() + telemetry.update_runtime(queue_depth=depth, queue_age=age, db_up=1, pool_used=used, pool_waiting=waiting) + except Exception: + telemetry.update_runtime(db_up=0) + try: + telemetry.update_runtime(worker_consumers=broker.consumer_count()) + except Exception: + telemetry.log_event("broker.stats_unavailable", severity="WARNING") + telemetry.sample_process() + try: + with db.connection() as conn: + broker.publish_pending(conn) + except Exception: + telemetry.log_event("job.publish_deferred", severity="WARNING") + + +def start(): + stop = threading.Event() + + def loop(): + while not stop.wait(2): + _once() + + _once() + threading.Thread(target=loop, name="poller", daemon=True).start() + return stop diff --git a/app/schema.sql b/app/schema.sql new file mode 100644 index 0000000..6ef564f --- /dev/null +++ b/app/schema.sql @@ -0,0 +1,108 @@ +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + expires_at timestamptz NOT NULL, + revoked_at timestamptz +); + +CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE INDEX IF NOT EXISTS messages_order_idx ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL, + sha256 text NOT NULL, + status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + error_code text, + staging_key text NOT NULL, + original_key text, + pipeline_version integer NOT NULL DEFAULT 1, + attempt integer NOT NULL DEFAULT 0, + lease_until timestamptz, + worker_id text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + ready_at timestamptz +); + +CREATE INDEX IF NOT EXISTS attachments_open_idx ON attachments (created_at) + WHERE status IN ('queued', 'processing'); + +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + name text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + pipeline_version integer NOT NULL, + PRIMARY KEY (attachment_id, name, pipeline_version) +); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + position integer NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); + +CREATE TABLE IF NOT EXISTS idempotency ( + user_id uuid NOT NULL, + method text NOT NULL, + path text NOT NULL, + idem_key text NOT NULL, + body_hash bytea NOT NULL, + status_code integer NOT NULL, + response_json jsonb NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, path, idem_key) +); + +CREATE TABLE IF NOT EXISTS outbox ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + published_at timestamptz +); + +CREATE INDEX IF NOT EXISTS outbox_unpublished_idx ON outbox (created_at) + WHERE published_at IS NULL; diff --git a/app/settings.py b/app/settings.py new file mode 100644 index 0000000..f540cb7 --- /dev/null +++ b/app/settings.py @@ -0,0 +1,45 @@ +import os +from pathlib import Path + + +def _read(name, file_name=None, default=None): + value = os.environ.get(name) + if value: + return value.strip() + file_env = os.environ.get(file_name) if file_name else None + if file_env: + return Path(file_env).read_text(encoding="utf-8").strip() + if default is not None: + return default + raise RuntimeError("missing configuration " + name) + + +class Settings: + def __init__(self): + self.service_name = os.environ.get("OTEL_SERVICE_NAME", "messenger-api") + self.service_version = os.environ.get("SERVICE_VERSION", "1.6.0") + self.instance_id = os.environ.get("INSTANCE_ID", "api-local") + self.environment = "lab" + self.database_url = _read("DATABASE_URL", "DATABASE_URL_FILE") + self.session_ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + self.idempotency_ttl = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) + self.s3_endpoint = os.environ.get("S3_ENDPOINT", "http://minio:9000") + self.s3_region = os.environ.get("S3_REGION", "us-east-1") + self.s3_bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + self.s3_access_key = _read("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE") + self.s3_secret_key = _read("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE") + self.broker_url = _read("BROKER_URL", "BROKER_URL_FILE") + self.worker_concurrency = int(os.environ.get("WORKER_CONCURRENCY", "2")) + self.job_max_attempts = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) + self.job_timeout = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) + self.staging_retention = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400")) + self.max_upload = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) + self.max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) + self.pool_max = int(os.environ.get("DB_POOL_MAX", "8")) + self.otlp_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") + self.fault_dir = os.environ.get("FAULT_DIR", "/faults") + self.rabbit_management_url = os.environ.get("RABBITMQ_MANAGEMENT_URL", "http://rabbitmq:15672") + self.role = os.environ.get("APP_ROLE", "api") + + +settings = Settings() diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 0000000..93089c6 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,92 @@ +import threading + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from app import telemetry +from app.settings import settings + +_client = None +_lock = threading.Lock() + + +def init(): + global _client + _client = boto3.client( + "s3", + endpoint_url=settings.s3_endpoint, + aws_access_key_id=settings.s3_access_key, + aws_secret_access_key=settings.s3_secret_key, + region_name=settings.s3_region, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=2, + read_timeout=10, + retries={"max_attempts": 2, "mode": "standard"}, + ), + ) + try: + _client.head_bucket(Bucket=settings.s3_bucket) + return + except ClientError as exc: + code = str(exc.response.get("Error", {}).get("Code", "")) + if code not in {"404", "NoSuchBucket", "NotFound"}: + raise + try: + _client.create_bucket(Bucket=settings.s3_bucket) + except ClientError as exc: + code = str(exc.response.get("Error", {}).get("Code", "")) + if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}: + raise + _warm() + + +def _warm(): + """Read one existing object so the first user download is not a cold volume open.""" + try: + listed = _client.list_objects_v2(Bucket=settings.s3_bucket, MaxKeys=1) + except (BotoCoreError, ClientError, OSError): + return + contents = listed.get("Contents") or [] + if not contents: + return + try: + body = _client.get_object(Bucket=settings.s3_bucket, Key=contents[0]["Key"])["Body"] + body.read() + except (BotoCoreError, ClientError, OSError): + return + + +def ready(): + try: + with _lock: + _client.head_bucket(Bucket=settings.s3_bucket) + return True + except (BotoCoreError, ClientError, OSError): + return False + + +def put(key, data, content_type, variant): + with telemetry.tracer().start_as_current_span( + "s3.put", + attributes={"s3.operation": "put", "variant": variant}, + ): + with _lock: + _client.put_object(Bucket=settings.s3_bucket, Key=key, Body=data, ContentType=content_type) + + +def get(key, variant): + with telemetry.tracer().start_as_current_span( + "s3.get", + attributes={"s3.operation": "get", "variant": variant}, + ): + with _lock: + response = _client.get_object(Bucket=settings.s3_bucket, Key=key) + return response["Body"].read() + + +def delete(key): + with _lock: + _client.delete_object(Bucket=settings.s3_bucket, Key=key) diff --git a/app/telemetry.py b/app/telemetry.py new file mode 100644 index 0000000..1248eca --- /dev/null +++ b/app/telemetry.py @@ -0,0 +1,379 @@ +import json +import logging +import threading +import time + +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.propagate import extract, inject +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk._logs.export import LogExportResult +from opentelemetry.sdk.metrics.export import MetricExportResult +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult +from opentelemetry.trace import Status, StatusCode + +from app.settings import settings + +BANNED_FIELDS = { + "password", + "token", + "authorization", + "secret", + "filename", + "text", + "body", + "database_url", + "broker_url", + "image", +} + +_dropped = None +_http_requests = None +_http_duration = None +_jobs = None +_job_duration = None +_ready_duration = None +_db_duration = None +_state = { + "queue_depth": 0, + "queue_age": 0.0, + "db_up": 0, + "pool_used": 0, + "pool_waiting": 0, + "worker_busy": 0, + "worker_capacity": 0, + "worker_consumers": 0, + "cpu": 0.0, + "memory": 0, +} +_cpu_last = None +_lock = threading.Lock() + + +class _GuardedExporter: + def __init__(self, inner, signal): + self.inner = inner + self.signal = signal + + def __getattr__(self, name): + return getattr(self.inner, name) + + def export(self, items, **kwargs): + try: + result = self.inner.export(items) + except Exception: + _count_drop(self.signal, len(items)) + return self._failure() + if getattr(result, "name", None) != "SUCCESS": + _count_drop(self.signal, len(items)) + return result + + def _failure(self): + if self.signal == "metrics": + return MetricExportResult.FAILURE + if self.signal == "logs": + return LogExportResult.FAILURE + return SpanExportResult.FAILURE + + def shutdown(self, **kwargs): + try: + return self.inner.shutdown(**kwargs) + except Exception: + return None + + def force_flush(self, timeout_millis=3000): + try: + return self.inner.force_flush(timeout_millis) + except Exception: + return False + + +class _RawFormatter(logging.Formatter): + def format(self, record): + return record.getMessage() + + +def _count_drop(signal, count): + if _dropped is not None and count: + _dropped.add(count, {"signal": signal}) + + +def _resource(): + return Resource.create( + { + "service.name": settings.service_name, + "service.version": settings.service_version, + "service.instance.id": settings.instance_id, + "deployment.environment.name": settings.environment, + } + ) + + +def _histogram(name, buckets): + return View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(buckets)) + + +def init(): + global _dropped, _http_requests, _http_duration, _jobs, _job_duration, _ready_duration, _db_duration + resource = _resource() + endpoint = settings.otlp_endpoint + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor( + _GuardedExporter(OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=5), "spans"), + max_queue_size=256, + schedule_delay_millis=2000, + max_export_batch_size=64, + export_timeout_millis=5000, + ) + ) + trace.set_tracer_provider(tracer_provider) + + reader = PeriodicExportingMetricReader( + _GuardedExporter(OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=5), "metrics"), + export_interval_millis=5000, + export_timeout_millis=4000, + ) + meter_provider = MeterProvider( + resource=resource, + metric_readers=[reader], + views=[ + _histogram("http.server.duration", [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]), + _histogram("messenger.job.duration", [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]), + _histogram("messenger.attachment.time_to_ready", [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]), + _histogram("messenger.db.duration", [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]), + ], + ) + from opentelemetry import metrics + + metrics.set_meter_provider(meter_provider) + meter = metrics.get_meter("messenger", settings.service_version) + _dropped = meter.create_counter("messenger.telemetry.dropped", unit="1") + _http_requests = meter.create_counter("http.server.requests", unit="1") + _http_duration = meter.create_histogram("http.server.duration", unit="s") + _jobs = meter.create_counter("messenger.jobs", unit="1") + _job_duration = meter.create_histogram("messenger.job.duration", unit="s") + _ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s") + _db_duration = meter.create_histogram("messenger.db.duration", unit="s") + meter.create_observable_gauge("messenger.queue.depth", callbacks=[_queue_depth]) + meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[_queue_age], unit="s") + meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used]) + meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting]) + meter.create_observable_gauge("messenger.worker.busy", callbacks=[_worker_busy]) + meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_worker_capacity]) + meter.create_observable_gauge("messenger.worker.consumers", callbacks=[_worker_consumers]) + meter.create_observable_gauge("messenger.db.up", callbacks=[_db_up]) + meter.create_observable_gauge("messenger.process.cpu.utilization", callbacks=[_cpu]) + meter.create_observable_gauge("messenger.process.memory", callbacks=[_memory], unit="By") + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + _GuardedExporter(OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=5), "logs"), + max_queue_size=256, + schedule_delay_millis=2000, + max_export_batch_size=64, + export_timeout_millis=5000, + ) + ) + root = logging.getLogger() + root.handlers.clear() + root.setLevel(logging.INFO) + stream = logging.StreamHandler() + stream.setFormatter(_RawFormatter()) + root.addHandler(stream) + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + root.addHandler(otel_handler) + for name in ("pika", "botocore", "boto3", "urllib3", "uvicorn", "uvicorn.error", "uvicorn.access"): + logging.getLogger(name).setLevel(logging.WARNING) + quiet = logging.getLogger("opentelemetry") + quiet.handlers.clear() + quiet.propagate = False + quiet.setLevel(logging.CRITICAL) + log_event("service.started", role=settings.role) + + +def tracer(): + return trace.get_tracer("messenger", settings.service_version) + + +def current_ids(): + span = trace.get_current_span() + ctx = span.get_span_context() + if not ctx or not ctx.is_valid: + return None, None + return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x") + + +def log_event(event, severity="INFO", **fields): + trace_id, span_id = current_ids() + payload = { + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000000) % 1000000:06d}Z", + "severity": severity, + "service.name": settings.service_name, + "event": event, + "trace_id": trace_id, + "span_id": span_id, + } + for key, value in fields.items(): + if key in BANNED_FIELDS or value is None: + continue + if isinstance(value, (str, int, float, bool)): + payload[key] = value + level = getattr(logging, severity, logging.INFO) + logging.getLogger("messenger").log(level, json.dumps(payload, ensure_ascii=False)) + + +def inject_carrier(carrier=None): + target = {} if carrier is None else carrier + inject(target) + return target + + +def extract_context(carrier): + clean = {} + for key, value in (carrier or {}).items(): + if isinstance(value, bytes): + value = value.decode("utf-8", "replace") + if isinstance(value, str): + clean[str(key).lower()] = value + return extract(clean) + + +def record_http(method, route, status, duration): + if _http_requests is None: + return + klass = f"{int(status) // 100}xx" + attributes = { + "http.request.method": method, + "http.route": route, + "http.response.status_class": klass, + } + _http_requests.add(1, attributes) + _http_duration.record(duration, attributes) + + +def record_job(outcome, duration): + attributes = {"job_type": "attachment.process.v1", "outcome": outcome} + if _jobs is not None: + _jobs.add(1, attributes) + _job_duration.record(max(duration, 0.0), attributes) + + +def record_ready(duration): + if _ready_duration is not None: + _ready_duration.record(max(duration, 0.0), {"job_type": "attachment.process.v1"}) + + +def record_db(duration): + if _db_duration is not None: + _db_duration.record(max(duration, 0.0), {"db.system": "postgresql"}) + + +def update_runtime(**values): + with _lock: + _state.update(values) + + +def note_worker(busy, capacity): + update_runtime(worker_busy=busy, worker_capacity=capacity) + + +def sample_process(): + global _cpu_last + now_cpu = time.process_time() + now_wall = time.monotonic() + ratio = 0.0 + if _cpu_last is not None: + dc = now_cpu - _cpu_last[0] + dw = now_wall - _cpu_last[1] + if dw > 0: + ratio = dc / dw + _cpu_last = (now_cpu, now_wall) + memory = 0 + try: + with open("/proc/self/statm", encoding="ascii") as handle: + resident = int(handle.read().split()[1]) + memory = resident * os_page() + except (OSError, IndexError, ValueError): + memory = 0 + update_runtime(cpu=ratio, memory=memory) + + +_page = None + + +def os_page(): + global _page + if _page is None: + import os + + _page = os.sysconf("SC_PAGE_SIZE") + return _page + + +def _obs(value, attributes=None): + from opentelemetry.metrics import Observation + + return [Observation(value, attributes or {})] + + +def _queue_depth(_options): + return _obs(_state["queue_depth"], {"queue": "attachment.process.v1"}) + + +def _queue_age(_options): + return _obs(_state["queue_age"], {"queue": "attachment.process.v1"}) + + +def _pool_used(_options): + return _obs(_state["pool_used"], {"pool": "postgres"}) + + +def _pool_waiting(_options): + return _obs(_state["pool_waiting"], {"pool": "postgres"}) + + +def _worker_busy(_options): + if settings.role != "worker": + return [] + return _obs(_state["worker_busy"], {"queue": "attachment.process.v1"}) + + +def _worker_capacity(_options): + if settings.role != "worker": + return [] + return _obs(_state["worker_capacity"], {"queue": "attachment.process.v1"}) + + +def _worker_consumers(_options): + return _obs(_state["worker_consumers"], {"queue": "attachment.process.v1"}) + + +def _db_up(_options): + return _obs(_state["db_up"]) + + +def _cpu(_options): + return _obs(_state["cpu"]) + + +def _memory(_options): + return _obs(_state["memory"]) + + +def mark_error(span): + span.set_status(Status(StatusCode.ERROR)) + + +def attach(ctx): + return otel_context.attach(ctx) diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 0000000..0d70f78 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,347 @@ +import datetime as dt +import hashlib +import json +import signal +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pika +from botocore.exceptions import BotoCoreError, ClientError +from opentelemetry.trace import SpanKind + +from app import broker, db, storage, telemetry +from app.broker import QUEUE, _params +from app.codec import utcnow +from app.errors import AppError +from app.faults import consume_process_failure +from app.images import thumbnail_jpeg +from app.poller import start +from app.settings import settings + +_inflight = 0 +_inflight_lock = threading.Lock() +_stop = threading.Event() + + +class Transient(Exception): + pass + + +class Permanent(Exception): + pass + + +@contextmanager +def _busy(): + global _inflight + with _inflight_lock: + _inflight += 1 + current = _inflight + telemetry.note_worker(current, settings.worker_concurrency) + try: + yield + finally: + with _inflight_lock: + _inflight -= 1 + current = _inflight + telemetry.note_worker(current, settings.worker_concurrency) + + +def _mark_failed(attachment_id, attempt, code): + with db.connection() as conn: + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = %s, lease_until = NULL, worker_id = NULL, updated_at = %s + WHERE id = %s AND status = 'processing' AND attempt = %s + """, + (code, utcnow(), attachment_id, attempt), + ) + telemetry.log_event("job.failed", attachment_id=str(attachment_id), attempt=attempt, error_code=code) + + +def _release(attachment_id, attempt): + with db.connection() as conn: + conn.execute( + """ + UPDATE attachments + SET status = 'queued', lease_until = NULL, worker_id = NULL, updated_at = %s + WHERE id = %s AND status = 'processing' AND attempt = %s + """, + (utcnow(), attachment_id, attempt), + ) + + +def process_delivery(body, headers): + started = time.perf_counter() + try: + payload = json.loads(body.decode("utf-8")) + attachment_id = uuid.UUID(str(payload["attachment_id"])) + pipeline = int(payload["pipeline_version"]) + except (ValueError, UnicodeError, KeyError, TypeError): + telemetry.log_event("job.poison", severity="ERROR") + return "ack" + carrier = {} + for key, value in (headers or {}).items(): + if isinstance(value, bytes): + value = value.decode("utf-8", "replace") + if isinstance(value, str): + carrier[str(key).lower()] = value + if payload.get("traceparent"): + carrier.setdefault("traceparent", payload["traceparent"]) + if payload.get("tracestate"): + carrier.setdefault("tracestate", payload["tracestate"]) + parent = telemetry.extract_context(carrier) + outcome = None + with telemetry.tracer().start_as_current_span( + "attachment.process consume", + context=parent, + kind=SpanKind.CONSUMER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": QUEUE, + "messaging.operation.type": "consume", + }, + ) as span: + try: + action, outcome, attempt = _consume(payload, attachment_id, pipeline, span) + except Exception: + telemetry.mark_error(span) + telemetry.log_event("job.failed", severity="ERROR", error_type="unexpected", attachment_id=str(attachment_id)) + action, outcome = "requeue", "retry" + attempt = None + if outcome: + telemetry.record_job(outcome, time.perf_counter() - started) + if attempt is not None: + span.set_attribute("attempt", attempt) + if outcome in {"failed", "retry"}: + telemetry.mark_error(span) + return action + + +def _consume(payload, attachment_id, pipeline, span): + now = utcnow() + with db.connection() as conn: + row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone() + if not row or int(row["pipeline_version"]) != pipeline: + return "ack", None, None + if row["status"] == "ready": + return "ack", None, int(row["attempt"]) + if row["status"] == "failed": + return "ack", None, int(row["attempt"]) + if row["status"] == "processing" and row["lease_until"] and row["lease_until"] > now: + held_attempt = int(row["attempt"]) + staging_key = None + else: + held_attempt = None + attempt = int(row["attempt"]) + 1 + if attempt > settings.job_max_attempts: + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = 'retry_exhausted', lease_until = NULL, updated_at = %s + WHERE id = %s + """, + (now, attachment_id), + ) + return "ack", "failed", attempt + lease = now + dt.timedelta(seconds=settings.job_timeout) + conn.execute( + """ + UPDATE attachments + SET status = 'processing', attempt = %s, lease_until = %s, worker_id = %s, updated_at = %s + WHERE id = %s + """, + (attempt, lease, settings.instance_id, now, attachment_id), + ) + staging_key = row["staging_key"] + content_type = row["content_type"] + expected_sha = row["sha256"] + created_at = row["created_at"] + if staging_key is None: + span.add_event("lease_held") + time.sleep(1) + return "requeue", None, held_attempt + span.set_attribute("attempt", attempt) + telemetry.log_event("job.consumed", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id"))) + try: + if consume_process_failure(): + span.add_event("retry", {"reason": "injected"}) + raise Transient() + data = storage.get(staging_key, "staging") + if hashlib.sha256(data).hexdigest() != expected_sha: + raise Permanent("invalid_image") + thumb, width, height = thumbnail_jpeg(data) + original_key = f"originals/{attachment_id}" + thumb_key = f"variants/{attachment_id}/thumbnail" + storage.put(original_key, data, content_type, "original") + storage.put(thumb_key, thumb, "image/jpeg", "thumbnail") + except Permanent as exc: + _mark_failed(attachment_id, attempt, str(exc) or "processing_failed") + return "ack", "failed", attempt + except AppError as exc: + code = "invalid_image" if exc.status in {413, 415} else "processing_failed" + _mark_failed(attachment_id, attempt, code) + return "ack", "failed", attempt + except Transient: + _release(attachment_id, attempt) + telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="transient") + time.sleep(0.5) + return "requeue", "retry", attempt + except (BotoCoreError, ClientError, OSError, TimeoutError): + _release(attachment_id, attempt) + telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="storage") + time.sleep(min(2 ** max(attempt - 1, 0), 8)) + return "requeue", "retry", attempt + thumb_sha = hashlib.sha256(thumb).hexdigest() + ready_at = utcnow() + with db.connection() as conn: + current = conn.execute("SELECT status, attempt FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone() + if not current or current["status"] != "processing" or int(current["attempt"]) != attempt: + return "ack", None, attempt + conn.execute( + """ + INSERT INTO attachment_variants ( + attachment_id, name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version + ) VALUES (%s, 'thumbnail', 'image/jpeg', %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, name, pipeline_version) DO UPDATE + SET content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256, + object_key = EXCLUDED.object_key + """, + (attachment_id, len(thumb), width, height, thumb_sha, thumb_key, pipeline), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'ready', error_code = NULL, original_key = %s, ready_at = %s, + lease_until = NULL, updated_at = %s + WHERE id = %s AND status = 'processing' AND attempt = %s + """, + (original_key, ready_at, ready_at, attachment_id, attempt), + ) + try: + storage.delete(staging_key) + except (BotoCoreError, ClientError, OSError): + telemetry.log_event("staging.cleanup_deferred", severity="WARNING", attachment_id=str(attachment_id)) + telemetry.record_ready((ready_at - created_at).total_seconds()) + telemetry.log_event("job.ready", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id"))) + return "ack", "success", attempt + + +def _serve(connection): + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True) + channel.basic_qos(prefetch_count=settings.worker_concurrency) + executor = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job") + + def on_message(ch, method, properties, body): + def work(): + with _busy(): + try: + action = process_delivery(body, properties.headers if properties else None) + except Exception: + action = "requeue" + + def finish(): + try: + if action == "ack": + ch.basic_ack(method.delivery_tag) + else: + ch.basic_nack(method.delivery_tag, requeue=True) + except Exception: + telemetry.log_event("job.ack_failed", severity="ERROR") + + connection.add_callback_threadsafe(finish) + + executor.submit(work) + + channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False) + + def watch(): + _stop.wait() + connection.add_callback_threadsafe(channel.stop_consuming) + + threading.Thread(target=watch, name="stop-watch", daemon=True).start() + channel.start_consuming() + executor.shutdown(wait=True, cancel_futures=False) + + +class _Health(BaseHTTPRequestHandler): + def do_GET(self): + if self.path.startswith("/health/live"): + self._send(200) + return + if self.path.startswith("/health/ready"): + try: + with db.connection() as conn: + conn.execute("SELECT 1") + ok = storage.ready() and broker.ready() + except Exception: + ok = False + self._send(200 if ok else 503) + return + self._send(404) + + def _send(self, status): + body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}' + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + return + + +def _health_server(): + server = ThreadingHTTPServer(("0.0.0.0", 8081), _Health) + server.serve_forever() + + +def _request_stop(signum, _frame): + _stop.set() + + +def main(): + signal.signal(signal.SIGTERM, _request_stop) + signal.signal(signal.SIGINT, _request_stop) + telemetry.init() + from app.main import _retry + + _retry("postgres", db.init) + db.migrate() + _retry("s3", storage.init) + _retry("broker", broker.init) + telemetry.note_worker(0, settings.worker_concurrency) + start() + threading.Thread(target=_health_server, name="health", daemon=True).start() + while not _stop.is_set(): + connection = None + try: + connection = pika.BlockingConnection(_params()) + telemetry.update_runtime(worker_consumers=settings.worker_concurrency) + _serve(connection) + except Exception: + if _stop.is_set(): + break + telemetry.log_event("broker.reconnect", severity="WARNING", error_type="connection") + time.sleep(1) + finally: + if connection is not None and connection.is_open: + try: + connection.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..cd3f879 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,236 @@ +name: mtusi-lab6 + +x-app-env: &app-env + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_KEY} + BROKER_URL: amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672/%2F + RABBITMQ_MANAGEMENT_URL: http://rabbitmq:15672 + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + DB_POOL_MAX: "8" + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + SERVICE_VERSION: "1.6.0" + FAULT_DIR: /faults + AWS_EC2_METADATA_DISABLED: "true" + +services: + postgres: + image: postgres:16.6-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: messenger + volumes: + - pg-data:/var/lib/postgresql/data + - ./app/schema.sql:/docker-entrypoint-initdb.d/01-schema.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + mem_limit: 512m + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs:4.51 + command: ["server", "-dir=/data", "-ip=0.0.0.0", "-ip.bind=0.0.0.0", "-s3", "-s3.port=8333", "-s3.ip.bind=0.0.0.0", "-s3.config=/etc/seaweed/s3.json"] + volumes: + - s3-data:/data + - ./infra/seaweed/s3.json:/etc/seaweed/s3.json:ro + healthcheck: + test: ["CMD", "weed", "version"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 5s + mem_limit: 512m + restart: unless-stopped + + rabbitmq: + image: rabbitmq:3.13.7-management-alpine + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD} + volumes: + - rabbit-data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 15s + mem_limit: 512m + restart: unless-stopped + + api-1: &api + build: . + image: mtusi-lab6-messenger:1.6.0 + environment: + <<: *app-env + OTEL_SERVICE_NAME: messenger-api + INSTANCE_ID: api-1 + APP_ROLE: api + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.6.0,service.instance.id=api-1 + volumes: + - ./faults:/faults + depends_on: + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + s3: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 5s + timeout: 4s + retries: 30 + start_period: 20s + mem_limit: 512m + restart: unless-stopped + + api-2: + <<: *api + environment: + <<: *app-env + OTEL_SERVICE_NAME: messenger-api + INSTANCE_ID: api-2 + APP_ROLE: api + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.6.0,service.instance.id=api-2 + + worker: + build: . + image: mtusi-lab6-messenger:1.6.0 + command: ["python", "-m", "app.worker"] + environment: + <<: *app-env + OTEL_SERVICE_NAME: messenger-worker + INSTANCE_ID: worker-1 + APP_ROLE: worker + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.6.0,service.instance.id=worker-1 + volumes: + - ./faults:/faults + depends_on: + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + s3: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=3)"] + interval: 5s + timeout: 4s + retries: 30 + start_period: 20s + mem_limit: 512m + restart: unless-stopped + + nginx: + image: nginx:1.27.4-alpine + depends_on: + api-1: + condition: service_healthy + api-2: + condition: service_healthy + ports: + - "127.0.0.1:8080:80" + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro + mem_limit: 128m + restart: unless-stopped + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.120.0 + command: ["--config=/etc/otelcol/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro + depends_on: + - tempo + - loki + - prometheus + mem_limit: 320m + restart: unless-stopped + + prometheus: + image: prom/prometheus:v2.55.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=2d + - --web.enable-lifecycle + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules.yml:/etc/prometheus/rules.yml:ro + - prometheus-data:/prometheus + ports: + - "127.0.0.1:9090:9090" + mem_limit: 512m + restart: unless-stopped + + loki: + image: grafana/loki:3.3.2 + command: ["-config.file=/etc/loki/loki.yml"] + volumes: + - ./infra/loki/loki.yml:/etc/loki/loki.yml:ro + - loki-data:/loki + ports: + - "127.0.0.1:3100:3100" + mem_limit: 512m + restart: unless-stopped + + tempo: + image: grafana/tempo:2.7.2 + command: ["-config.file=/etc/tempo/tempo.yml"] + volumes: + - ./infra/tempo/tempo.yml:/etc/tempo/tempo.yml:ro + - tempo-data:/var/tempo + ports: + - "127.0.0.1:3200:3200" + mem_limit: 512m + restart: unless-stopped + + grafana: + image: grafana/grafana:11.5.2 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_SERVER_HTTP_PORT: "3000" + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + ports: + - "127.0.0.1:3000:3000" + depends_on: + - prometheus + - loki + - tempo + mem_limit: 384m + restart: unless-stopped + +volumes: + pg-data: + s3-data: + rabbit-data: + prometheus-data: + loki-data: + tempo-data: + grafana-data: diff --git a/docs/runbooks/api-error-budget.md b/docs/runbooks/api-error-budget.md new file mode 100644 index 0000000..ee152e2 --- /dev/null +++ b/docs/runbooks/api-error-budget.md @@ -0,0 +1,18 @@ +# API error budget + +SLO: 99% of business HTTP responses are not 5xx, measured on routes other than `/health/*`. +4xx is a client outcome and is outside the numerator. The ratio is `5xx / all business requests`. + +Windows used on this lab stand: + +- fast burn: 1 minute and 5 minutes, both above 14.4% (14.4 times a 1% budget), `for` 30s, and at least 0.2 requests/s +- slow burn: 5 minutes and 15 minutes, both above 6%, `for` 1 minute +- simple page: 5 minute ratio above 5% for 1 minute + +A short spike can move the 1 minute window without moving the 5 minute window enough, so the fast burn stays inactive. A sustained burst moves both. + +1. Open Prometheus `ALERTS` or the Messenger dashboard panel "Prometheus alert state". +2. If `ApiErrorBudgetFastBurn` or `ApiHighErrorRate` is firing, open a trace for a 5xx request. +3. If the `db.transaction` span is the long or erroring child, follow `docs/runbooks/db-latency.md`. +4. If the server span is long and the database span is short, the handler itself is slow (`handler_slow_ms` fault or application code). +5. Remove the fault file and wait for the next evaluations. The alert leaves `firing` when the expression is false; `for` does not delay resolution. diff --git a/docs/runbooks/db-latency.md b/docs/runbooks/db-latency.md new file mode 100644 index 0000000..64b04ef --- /dev/null +++ b/docs/runbooks/db-latency.md @@ -0,0 +1,15 @@ +# Slow or unavailable database + +`messenger_db_up` is 0 when the process cannot run `SELECT 1`. Business calls then return 503 `unavailable`. That increases the API error ratio. Health `/health/live` stays 200. `/health/ready` becomes 503. + +A slow database is different from a slow handler: + +- slow database: the `db.transaction` span is long and the server span is about the same length +- slow handler: `handler_slow` sits in the server span, and `db.transaction` stays short + +`DatabaseUnready` fires when `min(messenger_db_up) == 0` for 30 seconds. + +1. Compare one server span with its `db.transaction` child in Tempo. +2. If the database span failed, check PostgreSQL. Grafana being down does not cause this alert. +3. Restore PostgreSQL. `messenger_db_up` returns to 1 and the error-budget alerts resolve after the windows slide. +4. The collector and Grafana are not on the readiness path. diff --git a/docs/runbooks/queue-lag.md b/docs/runbooks/queue-lag.md new file mode 100644 index 0000000..406e451 --- /dev/null +++ b/docs/runbooks/queue-lag.md @@ -0,0 +1,14 @@ +# Queue lag and worker loss + +`messenger_queue_oldest_age_seconds` is the age of the oldest attachment still `queued` or `processing`. It is not the duration of a completed job. `messenger_queue_depth` is how many such rows exist. Both are read from PostgreSQL by the API, so they keep moving when the worker process is stopped. + +`messenger_worker_consumers` is the RabbitMQ consumer count. The API polls the management API, so the series becomes 0 instead of going stale. + +Alert `QueueLagHigh`: max age above 30 seconds for 1 minute. +Alert `WorkerConsumersZero`: max consumers equals 0 for 1 minute. + +1. Confirm consumers are 0 or age is climbing on the Messenger dashboard. +2. Check worker logs in Loki for `job.failed` or `broker.reconnect`. A clean stop has neither new `job.consumed` nor a database error. +3. Start the worker again: `docker compose --env-file .env start worker`. +4. Age returns to 0 after the backlog is acknowledged. Alerts resolve on the next evaluation. +5. Do not use p95 time-to-ready alone: it only observes jobs that reached `ready`. diff --git a/docs/runbooks/telemetry-backend.md b/docs/runbooks/telemetry-backend.md new file mode 100644 index 0000000..fb6559e --- /dev/null +++ b/docs/runbooks/telemetry-backend.md @@ -0,0 +1,8 @@ +# Telemetry backend unavailable + +The API and worker export OTLP on a background queue (256 spans or log records, 5 second timeout). The collector limits itself to 192 MiB, batches, and keeps a sending queue of 200 items with retry capped at 30 seconds. When Tempo, Loki, or Prometheus is down, exports fail and `messenger_telemetry_dropped_total` increases. Message acceptance does not wait on those exports. + +1. Stop one backend, for example `docker compose --env-file .env stop tempo`. +2. Send a few API requests. `/health/ready` stays 200 and messages still return 201. +3. Watch collector memory. It should stay near its limit, not grow without bound. +4. Start the backend again. New spans appear. Spans dropped during the outage are not replayed from the in-memory queue once it has overflowed. diff --git a/docs/runbooks/time-to-ready.md b/docs/runbooks/time-to-ready.md new file mode 100644 index 0000000..833e5f3 --- /dev/null +++ b/docs/runbooks/time-to-ready.md @@ -0,0 +1,11 @@ +# Time to ready + +SLO: 95% of attachments that reach `ready` do so within 10 seconds of durable acceptance. +The series is `messenger_attachment_time_to_ready_seconds`, a histogram in seconds. +`AttachmentReadyLatencyHigh` uses `histogram_quantile(0.95, ...)` over 15 minutes and pages after 2 minutes above 10 seconds. + +This SLO does not see jobs that never finish. Pair it with queue age. + +1. If p95 is high and queue age is low, open a worker trace and compare `s3.put`, `s3.get`, and `db.transaction`. +2. A retry span with `attempt > 1` and event `retry` means the first attempt was transient. The successful attempt should still contain S3 and the ready update. +3. If p95 is quiet but users wait, read queue age. The worker is likely stopped or stuck before completion. diff --git a/evidence/incidents.json b/evidence/incidents.json new file mode 100644 index 0000000..d0e7c64 --- /dev/null +++ b/evidence/incidents.json @@ -0,0 +1,78 @@ +{ + "alerts_before": { + "AttachmentReadyLatencyHigh": "pending" + }, + "while_worker_stopped": { + "first_firing": { + "WorkerConsumersZero": 75.5, + "QueueLagHigh": 105.7 + }, + "samples": [ + { + "t": 90.6, + "alerts": { + "QueueLagHigh": "pending", + "AttachmentReadyLatencyHigh": "firing", + "WorkerConsumersZero": "firing" + }, + "age": "82.87327", + "api_consumers": "0" + }, + { + "t": 95.6, + "alerts": { + "QueueLagHigh": "pending", + "AttachmentReadyLatencyHigh": "firing", + "WorkerConsumersZero": "firing" + }, + "age": "88.417685", + "api_consumers": "0" + }, + { + "t": 100.7, + "alerts": { + "QueueLagHigh": "pending", + "AttachmentReadyLatencyHigh": "firing", + "WorkerConsumersZero": "firing" + }, + "age": "92.908538", + "api_consumers": "0" + }, + { + "t": 105.7, + "alerts": { + "QueueLagHigh": "firing", + "AttachmentReadyLatencyHigh": "firing", + "WorkerConsumersZero": "firing" + }, + "age": "98.453736", + "api_consumers": "0" + } + ] + }, + "after_worker_start": { + "cleared_after_seconds": { + "QueueLagHigh": 20.1, + "WorkerConsumersZero": 20.1 + }, + "alerts": { + "AttachmentReadyLatencyHigh": "firing" + } + }, + "tempo_down_40s": { + "api_results": { + "200": 76 + }, + "failed_before": [ + "9" + ], + "failed_after": [ + "24" + ], + "queue_size": [ + "0", + "22" + ], + "collector_memory": "68.8MiB / 320MiB" + } +} \ No newline at end of file diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..66ddcca --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.078, + "concurrency": 2, + "successful_requests": 1247, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 82.703, + "successful_latency_ms": { + "p50": 31.0, + "p95": 47.0, + "p99": 47.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/retry-trace.json b/evidence/retry-trace.json new file mode 100644 index 0000000..c1b9723 --- /dev/null +++ b/evidence/retry-trace.json @@ -0,0 +1,16 @@ +{ + "trace_id": "cf5025ad9d9a2a248da8335a0c6caee2", + "log": { + "job.retry": {"attempt": 1, "same_trace": true}, + "job.ready": {"attempt": 2, "same_trace": true} + }, + "chain": [ + "POST /api/v1/chats/{chat_id}/attachments", + "attachment.process publish parent=POST", + "attachment.process consume (attempt 1, injected transient) parent=publish", + "attachment.process consume (attempt 2) parent=publish", + "s3.get and s3.put parent=second consume", + "db.transaction parent=POST and parent=consume" + ], + "note": "Both consumer spans share the producer span as parent. No message text, filename, or object key is stored here." +} diff --git a/evidence/slo-track2.json b/evidence/slo-track2.json new file mode 100644 index 0000000..8429e1a --- /dev/null +++ b/evidence/slo-track2.json @@ -0,0 +1,89 @@ +{ + "started": "2026-11-06T21:40:04Z", + "retry_upload": { + "final_status": "ready", + "log": { + "event": "job.ready", + "attempt": 1, + "trace_id": "40c8e289c22994a3c786030354799554", + "span_id": "606017e0d8c6f6b5", + "service": "messenger-worker" + }, + "consume_spans": 1, + "span_names": [ + "POST /api/v1/chats/{chat_id}/attachments", + "db.transaction", + "db.transaction", + "s3.put", + "db.transaction", + "db.transaction", + "attachment.process publish", + "attachment.process consume", + "db.transaction", + "s3.get", + "s3.put", + "s3.put", + "db.transaction" + ], + "parents_linked": true + }, + "warmup": { + "200": 180 + }, + "alerts_after_warmup": {}, + "short_spike": { + "500": 20 + }, + "alerts_after_short_spike": { + "ApiHighErrorRate": "pending", + "ApiErrorBudgetSlowBurn": "pending" + }, + "recovery_after_spike": { + "200": 40 + }, + "fast_burn_detected_after_seconds": 51.6, + "alerts_when_fast_burn_fired": { + "ApiHighErrorRate": "firing", + "ApiErrorBudgetSlowBurn": "firing", + "ApiErrorBudgetFastBurn": "firing" + }, + "sustained_counts_until_detection_or_timeout": { + "500": 104 + }, + "fast_burn_resolved_after_seconds": 70.6, + "alerts_after_resolve": { + "ApiHighErrorRate": "firing", + "ApiErrorBudgetSlowBurn": "firing" + }, + "worker_stopped": { + "consumers_firing_after_seconds": null, + "queue_lag_firing_after_seconds": 0.0, + "alerts": { + "ApiHighErrorRate": "firing", + "ApiErrorBudgetSlowBurn": "firing", + "QueueLagHigh": "firing" + } + }, + "worker_restored": { + "consumers_cleared_after_seconds": 0.0, + "alerts": { + "ApiHighErrorRate": "firing", + "ApiErrorBudgetSlowBurn": "firing", + "QueueLagHigh": "firing" + } + }, + "latency_note": "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span", + "tempo_stopped": { + "api_results": { + "200": 16 + }, + "failed_spans_before": [ + "0" + ], + "failed_spans_after": [ + "0" + ], + "collector_memory": "67.75MiB / 320MiB" + }, + "finished": "2026-11-06T21:48:01Z" +} \ No newline at end of file diff --git a/faults/README.md b/faults/README.md new file mode 100644 index 0000000..d78829e --- /dev/null +++ b/faults/README.md @@ -0,0 +1,11 @@ +Local fault-injection files. They are not present by default, so the API behaves normally. + +Write one of these names in this directory while the stack is running: + +- `http_error_ratio` — float 0..1, fraction of `/api/` requests that return 500 +- `db_slow_ms` — delay added inside the `db.transaction` span +- `db_fail` — `1` makes database calls return 503 +- `handler_slow_ms` — delay inside the HTTP server span and outside the database span +- `process_fail_remaining` — positive integer, worker fails that many processing attempts once each + +Do not put secrets here. The worker must be able to update `process_fail_remaining`. diff --git a/infra/README.md b/infra/README.md index 5a39e86..4f0ea72 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,21 @@ -# Инфраструктура студента +# Инфраструктура -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Топология — корневой `compose.yaml`. `compose.example.yaml` остаётся worksheet курса и не запускается. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +| Роль | Сервис | Образ | Порт на хосте | Том | +| --- | --- | --- | --- | --- | +| PostgreSQL | postgres | postgres:16.6-alpine | нет | pg-data | +| S3 | s3 | chrislusf/seaweedfs:4.51 | нет | s3-data, плюс gitignored `infra/seaweed/s3.json` | +| Брокер | rabbitmq | rabbitmq:3.13.7-management-alpine | нет | rabbit-data | +| API | api-1, api-2 | сборка из Dockerfile | нет | `./faults` | +| Worker | worker | та же сборка | нет | `./faults` | +| Балансировщик | nginx | nginx:1.27.4-alpine | 127.0.0.1:8080 | — | +| Collector | otel-collector | otel/opentelemetry-collector-contrib:0.120.0 | нет | — | +| Метрики | prometheus | prom/prometheus:v2.55.1 | 127.0.0.1:9090 | prometheus-data | +| Логи | loki | grafana/loki:3.3.2 | 127.0.0.1:3100 | loki-data | +| Трассы | tempo | grafana/tempo:2.7.2 | 127.0.0.1:3200 | tempo-data | +| UI | grafana | grafana/grafana:11.5.2 | 127.0.0.1:3000 | grafana-data | -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Секреты приходят из `.env` (БД, S3, RabbitMQ, пароль Grafana). Идентичность SeaweedFS генерируется `python scripts/render-s3-config.py` и не коммитится. Схема БД — `app/schema.sql`. Бакет и очередь создают приложения при старте. + +Grafana не хранит метрики, логи и трассы. Datasources и дашборд `messenger-overview` импортируются из `infra/grafana/provisioning` и `infra/grafana/dashboards`. Правила алертов — `infra/prometheus/rules.yml`. diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..e10a727 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,187 @@ +{ + "uid": "messenger-overview", + "title": "Messenger observability", + "timezone": "utc", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "tags": ["messenger", "lab6"], + "time": {"from": "now-1h", "to": "now"}, + "panels": [ + { + "id": 1, + "type": "text", + "title": "Signal stores", + "gridPos": {"h": 4, "w": 24, "x": 0, "y": 0}, + "options": { + "mode": "markdown", + "content": "Prometheus stores metrics (rates, histograms, gauges). Loki stores structured logs. Tempo stores traces. The collector is not a database: it receives OTLP, redacts, batches, and exports. Grafana only queries those stores. Percentiles use `histogram_quantile` on cumulative buckets, summed across API replicas before the quantile." + } + }, + { + "id": 2, + "type": "timeseries", + "title": "API RPS by route", + "description": "Unit: requests per second. Sum of rates across replicas.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4}, + "fieldConfig": {"defaults": {"unit": "reqps", "custom": {"drawStyle": "line"}}, "overrides": []}, + "targets": [ + { + "refId": "A", + "expr": "sum by (http_route) (rate(http_server_requests_total{service_name=\"messenger-api\"}[1m]))", + "legendFormat": "{{http_route}}" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "API error ratio (5xx / business)", + "description": "Unit: fraction 0-1. Numerator is status class 5xx. Denominator is business routes, excluding /health and unmatched. 4xx is not an error for this SLI.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 4}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "messenger:api_error_ratio:rate5m", "legendFormat": "5m"}, + {"refId": "B", "expr": "messenger:api_error_ratio:rate1m", "legendFormat": "1m"} + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "API duration p50/p95/p99", + "description": "Unit: seconds. histogram_quantile over the sum of bucket rates. Not an average of per-replica quantiles.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p50"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p95"}, + {"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p99"} + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Jobs by outcome", + "description": "Unit: jobs per second. outcome is success, failed, or retry. job_type is attachment.process.v1.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 12}, + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))", "legendFormat": "{{outcome}}"} + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Worker job duration p95", + "description": "Unit: seconds, from the job histogram.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 20}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(messenger_job_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}"} + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Queue depth and oldest age", + "description": "Depth is a count of queued+processing attachments. Age is seconds since the oldest of those rows was accepted. max() avoids double-counting the two API replicas.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 20}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": [{"matcher": {"id": "byName", "options": "age"}, "properties": [{"id": "unit", "value": "s"}]}]}, + "targets": [ + {"refId": "A", "expr": "max(messenger_queue_depth{queue=\"attachment.process.v1\"})", "legendFormat": "depth"}, + {"refId": "B", "expr": "max(messenger_queue_oldest_age_seconds{queue=\"attachment.process.v1\"})", "legendFormat": "age"} + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Time to ready p95", + "description": "Unit: seconds from durable acceptance to ready. Completed jobs only.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 20}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[15m])))", "legendFormat": "p95"} + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "DB pool used and waiting", + "description": "Unit: connections. Summed across processes. waiting is requests blocked on the pool.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 28}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "sum(messenger_db_pool_used)", "legendFormat": "used"}, + {"refId": "B", "expr": "sum(messenger_db_pool_waiting)", "legendFormat": "waiting"} + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Worker saturation and consumers", + "description": "busy/capacity are counts inside the worker. consumers is the RabbitMQ consumer count observed by the API.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 28}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "max(messenger_worker_busy)", "legendFormat": "busy"}, + {"refId": "B", "expr": "max(messenger_worker_capacity)", "legendFormat": "capacity"}, + {"refId": "C", "expr": "max(messenger_worker_consumers{service_name=\"messenger-api\"})", "legendFormat": "consumers"} + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Process CPU and memory", + "description": "CPU unit is process CPU-seconds per wall second (can exceed 1). Memory unit is bytes of RSS.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 28}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "max by (service_name) (messenger_process_cpu_utilization)", "legendFormat": "cpu {{service_name}}"}, + {"refId": "B", "expr": "max by (service_name) (messenger_process_memory_bytes)", "legendFormat": "rss {{service_name}}"} + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "DB duration p95", + "description": "Unit: seconds. Includes an injected db_slow fault. Compare with the server span to separate a slow database from a slow handler.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 36}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_db_duration_seconds_bucket[5m])))", "legendFormat": "db p95"} + ] + }, + { + "id": 13, + "type": "table", + "title": "Prometheus alert state", + "description": "Local firing/pending state. No external notification channel is configured.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 36}, + "targets": [ + {"refId": "A", "expr": "ALERTS{alertstate=~\"firing|pending\"}", "instant": true, "format": "table"} + ] + }, + { + "id": 14, + "type": "logs", + "title": "API and worker logs", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 10, "w": 24, "x": 0, "y": 44}, + "targets": [ + {"refId": "A", "expr": "{service_name=~\"messenger-api|messenger-worker\"}"} + ] + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yml b/infra/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..3ed9466 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yml b/infra/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..c63a642 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,41 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 5s + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + derivedFields: + - name: TraceID + matcherRegex: "\"trace_id\":\"([0-9a-f]+)\"" + url: "$${__value.raw}" + datasourceUid: tempo + urlDisplayLabel: View trace + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + lokiSearch: + datasourceUid: loki diff --git a/infra/loki/loki.yml b/infra/loki/loki.yml new file mode 100644 index 0000000..7dca0ce --- /dev/null +++ b/infra/loki/loki.yml @@ -0,0 +1,39 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + volume_enabled: true + reject_old_samples: false + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + max_query_series: 500 + +analytics: + reporting_enabled: false diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 0000000..5e5b4ab --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,22 @@ +upstream messenger_api { + server api-1:8080 max_fails=1 fail_timeout=5s; + server api-2:8080 max_fails=1 fail_timeout=5s; +} + +server { + listen 80; + client_max_body_size 12m; + + proxy_http_version 1.1; + proxy_connect_timeout 1s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + proxy_next_upstream error timeout http_502 http_503 non_idempotent; + proxy_next_upstream_tries 2; + + location / { + proxy_pass http://messenger_api; + proxy_set_header Host $host; + proxy_set_header Connection ""; + } +} diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..e0a4b32 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -1,7 +1,15 @@ -# Конфигурация наблюдаемости — реализовать +# Collector -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. +`collector.yaml` принимает OTLP gRPC `:4317` и HTTP `:4318` только внутри сети Compose. -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. +Процессоры на всех трёх pipeline: memory limiter (192 MiB, spike 48 MiB), удаление URL, SQL и заголовков Authorization/Cookie, batch. -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. +Экспорт: + +- трассы — OTLP gRPC в Tempo, очередь 200, retry не дольше 30 с; +- логи — OTLP HTTP в Loki `/otlp/v1/logs`, та же очередь и retry; +- метрики — Prometheus exporter `:8889`, его скрейпит Prometheus. + +Внутренние метрики Collector — `:8888`. Health — `:13133`. Эти порты наружу не опубликованы. + +Если Tempo или Loki недоступны, очередь exporter ограничена 200, retry обрывается через 30 с, memory limiter не даёт процессу расти до лимита контейнера 320 MiB. Приложение при этом отвечает: экспорт не блокирует приём сообщений. diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..3ed9d3b --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,95 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + attributes/redact: + actions: + - key: url.full + action: delete + - key: url.query + action: delete + - key: url.path + action: delete + - key: http.url + action: delete + - key: http.target + action: delete + - key: db.statement + action: delete + - key: db.query.text + action: delete + - key: http.request.header.authorization + action: delete + - key: http.request.header.cookie + action: delete + batch: + timeout: 2s + send_batch_size: 256 + send_batch_max_size: 512 + +exporters: + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + timeout: 5s + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 15s + max_elapsed_time: 30s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 + otlphttp/loki: + logs_endpoint: http://loki:3100/otlp/v1/logs + tls: + insecure: true + timeout: 5s + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 15s + max_elapsed_time: 30s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + metric_expiration: 180s + +service: + extensions: [health_check] + telemetry: + metrics: + address: 0.0.0.0:8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/redact, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, attributes/redact, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/redact, batch] + exporters: [otlphttp/loki] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..ce5120e --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + scrape_timeout: 4s + +rule_files: + - /etc/prometheus/rules.yml + +scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8889"] + - job_name: otel-collector-internal + static_configs: + - targets: ["otel-collector:8888"] diff --git a/infra/prometheus/rules.yml b/infra/prometheus/rules.yml new file mode 100644 index 0000000..daddf41 --- /dev/null +++ b/infra/prometheus/rules.yml @@ -0,0 +1,96 @@ +groups: + - name: messenger-sli + interval: 5s + rules: + - record: messenger:api_requests:rate1m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[1m])) + - record: messenger:api_5xx:rate1m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[1m])) + - record: messenger:api_error_ratio:rate1m + expr: messenger:api_5xx:rate1m / clamp_min(messenger:api_requests:rate1m, 1e-9) + - record: messenger:api_requests:rate5m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[5m])) + - record: messenger:api_5xx:rate5m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[5m])) + - record: messenger:api_error_ratio:rate5m + expr: messenger:api_5xx:rate5m / clamp_min(messenger:api_requests:rate5m, 1e-9) + - record: messenger:api_requests:rate15m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[15m])) + - record: messenger:api_5xx:rate15m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[15m])) + - record: messenger:api_error_ratio:rate15m + expr: messenger:api_5xx:rate15m / clamp_min(messenger:api_requests:rate15m, 1e-9) + + - name: messenger-alerts + interval: 5s + rules: + - alert: ApiHighErrorRate + expr: messenger:api_error_ratio:rate5m > 0.05 and messenger:api_requests:rate5m > 0.2 + for: 1m + labels: + severity: page + slo: api-availability + annotations: + summary: API business 5xx ratio is above 5% + description: Numerator is 5xx on business routes. Denominator is all business requests. Health checks and 4xx are excluded. Window 5m, for 1m. + runbook: docs/runbooks/api-error-budget.md + - alert: ApiErrorBudgetFastBurn + expr: messenger:api_error_ratio:rate1m > 0.144 and messenger:api_error_ratio:rate5m > 0.144 and messenger:api_requests:rate1m > 0.2 + for: 30s + labels: + severity: page + slo: api-availability + burn: fast + annotations: + summary: Fast multi-window burn of the 1% API error budget + description: Both the 1m and 5m error ratios exceed 14.4% (14.4x a 1% budget). A sub-minute spike does not satisfy the 5m window when healthy traffic is present. + runbook: docs/runbooks/api-error-budget.md + - alert: ApiErrorBudgetSlowBurn + expr: messenger:api_error_ratio:rate5m > 0.06 and messenger:api_error_ratio:rate15m > 0.06 and messenger:api_requests:rate5m > 0.2 + for: 1m + labels: + severity: ticket + slo: api-availability + burn: slow + annotations: + summary: Slow multi-window burn of the API error budget + description: 5m and 15m error ratios exceed 6% (6x a 1% budget). This catches a long mild degradation that the fast burn may miss. + runbook: docs/runbooks/api-error-budget.md + - alert: QueueLagHigh + expr: max(messenger_queue_oldest_age_seconds{queue="attachment.process.v1"}) > 30 + for: 1m + labels: + severity: page + slo: time-to-ready + annotations: + summary: Oldest attachment job is older than 30s + description: Backlog age is not processing latency. It grows when the worker is stopped or cannot finish jobs. Unit is seconds. + runbook: docs/runbooks/queue-lag.md + - alert: WorkerConsumersZero + expr: max(messenger_worker_consumers{service_name="messenger-api",queue="attachment.process.v1"}) == 0 + for: 1m + labels: + severity: page + annotations: + summary: RabbitMQ has no attachment consumers + description: The API polls the RabbitMQ management API, so this series stays fresh after the worker process stops. + runbook: docs/runbooks/queue-lag.md + - alert: AttachmentReadyLatencyHigh + expr: histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[15m]))) > 10 + for: 2m + labels: + severity: ticket + slo: time-to-ready + annotations: + summary: p95 time to ready is above 10s + description: Histogram of completed jobs only. A stalled queue does not move this SLO; use QueueLagHigh for that. + runbook: docs/runbooks/time-to-ready.md + - alert: DatabaseUnready + expr: min(messenger_db_up) == 0 + for: 30s + labels: + severity: page + annotations: + summary: An API or worker process cannot use PostgreSQL + description: Distinguish this from a slow handler by comparing the db.transaction span with the server span. + runbook: docs/runbooks/db-latency.md diff --git a/infra/seaweed/README.md b/infra/seaweed/README.md new file mode 100644 index 0000000..629a64c --- /dev/null +++ b/infra/seaweed/README.md @@ -0,0 +1,9 @@ +# SeaweedFS S3 identity + +`s3.json` is generated locally and is not committed: + +``` +python scripts/render-s3-config.py +``` + +The script reads `S3_ACCESS_KEY` and `S3_SECRET_KEY` from `.env` and writes this directory's identity file. Compose mounts it read-only at `/etc/seaweed/s3.json`. diff --git a/infra/tempo/tempo.yml b/infra/tempo/tempo.yml new file mode 100644 index 0000000..80ad013 --- /dev/null +++ b/infra/tempo/tempo.yml @@ -0,0 +1,34 @@ +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + grpc_listen_port: 9096 + log_level: warn + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + trace_idle_period: 10s + max_block_duration: 5m + +compactor: + compaction: + block_retention: 48h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces + +querier: + max_concurrent_queries: 4 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ec90ba9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +starlette==0.45.3 +uvicorn==0.34.0 +psycopg[binary,pool]==3.2.6 +argon2-cffi==23.1.0 +boto3==1.37.38 +pika==1.3.2 +Pillow==11.1.0 +opentelemetry-api==1.29.0 +opentelemetry-sdk==1.29.0 +opentelemetry-exporter-otlp-proto-http==1.29.0 diff --git a/scripts/incident_followup.py b/scripts/incident_followup.py new file mode 100644 index 0000000..647c9df --- /dev/null +++ b/scripts/incident_followup.py @@ -0,0 +1,109 @@ +"""Re-check worker-stop alerts and a Tempo outage after the rule fix.""" + +import json +import subprocess +import sys +import time +import urllib.parse +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client # noqa: E402 + +OUT = ROOT / "evidence" / "incidents.json" + + +def prom(expr): + url = "http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr) + with urllib.request.urlopen(url, timeout=10) as response: + return json.load(response)["data"]["result"] + + +def alerts(): + return { + item["metric"].get("alertname"): item["metric"].get("alertstate") + for item in prom("ALERTS") + } + + +def reload_rules(): + request = urllib.request.Request("http://127.0.0.1:9090/-/reload", method="POST") + with urllib.request.urlopen(request, timeout=10) as response: + return response.status + + +def main(): + print("reload", reload_rules(), flush=True) + client = Client("http://127.0.0.1:8080", timeout=8) + alice, _bob, _eve, chat = client.bootstrap() + report = {"alerts_before": alerts()} + subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT) + client.attachment(alice, chat) + started = time.monotonic() + seen = {} + samples = [] + while time.monotonic() - started < 130: + state = alerts() + age = prom('max(messenger_queue_oldest_age_seconds{service_name="messenger-api"})') + consumers = prom('max(messenger_worker_consumers{service_name="messenger-api"})') + sample = { + "t": round(time.monotonic() - started, 1), + "alerts": state, + "age": age[0]["value"][1] if age else None, + "api_consumers": consumers[0]["value"][1] if consumers else None, + } + samples.append(sample) + for name in ("QueueLagHigh", "WorkerConsumersZero"): + if name not in seen and state.get(name) == "firing": + seen[name] = sample["t"] + if len(seen) == 2: + break + time.sleep(5) + report["while_worker_stopped"] = {"first_firing": seen, "samples": samples[-4:]} + subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT) + cleared = {} + restore_started = time.monotonic() + while time.monotonic() - restore_started < 90: + state = alerts() + for name in ("QueueLagHigh", "WorkerConsumersZero"): + if name not in cleared and name not in state: + cleared[name] = round(time.monotonic() - restore_started, 1) + if "QueueLagHigh" in cleared and "WorkerConsumersZero" in cleared: + break + time.sleep(5) + report["after_worker_start"] = {"cleared_after_seconds": cleared, "alerts": alerts()} + + before_failed = prom("otelcol_exporter_send_failed_spans") + subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT) + counts = {} + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + try: + status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"] + except Exception: + status = "error" + counts[str(status)] = counts.get(str(status), 0) + 1 + time.sleep(0.5) + time.sleep(5) + memory = subprocess.check_output( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"], + text=True, + ).strip() + queue = prom("otelcol_exporter_queue_size") + failed = prom("otelcol_exporter_send_failed_spans") + subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT) + report["tempo_down_40s"] = { + "api_results": counts, + "failed_before": [item["value"][1] for item in before_failed], + "failed_after": [item["value"][1] for item in failed], + "queue_size": [item["value"][1] for item in queue], + "collector_memory": memory, + } + OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + print("wrote", OUT, flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/probe_content_logs.py b/scripts/probe_content_logs.py new file mode 100644 index 0000000..66afcbc --- /dev/null +++ b/scripts/probe_content_logs.py @@ -0,0 +1,13 @@ +import json +import urllib.parse +import urllib.request + +query = '{service_name="messenger-api"} |= "http.request.completed" |= "content"' +url = "http://127.0.0.1:3100/loki/api/v1/query_range?query=" + urllib.parse.quote(query) + "&limit=15" +data = json.load(urllib.request.urlopen(url, timeout=10)) +values = [] +for stream in data["data"]["result"]: + values.extend(stream["values"]) +print("n", len(values)) +for _, line in values[-8:]: + print(line[:500]) diff --git a/scripts/probe_retry.py b/scripts/probe_retry.py new file mode 100644 index 0000000..53a0805 --- /dev/null +++ b/scripts/probe_retry.py @@ -0,0 +1,31 @@ +"""Find the newest worker job.retry / job.ready lines and summarize that trace.""" + +import json +import urllib.parse +import urllib.request + +def lines(event): + query = '{service_name="messenger-worker"} |= "' + event + '"' + payload = json.load( + urllib.request.urlopen( + "http://127.0.0.1:3100/loki/api/v1/query_range?query=" + + urllib.parse.quote(query) + + "&limit=20", + timeout=10, + ) + ) + found = [] + for stream in payload["data"]["result"]: + for ts, line in stream["values"]: + try: + body = json.loads(line) + except ValueError: + continue + if body.get("event") == event: + found.append((int(ts), body.get("attempt"), body.get("trace_id"))) + found.sort() + return found[-5:] + + +print("retry", lines("job.retry")) +print("ready", lines("job.ready")) diff --git a/scripts/probe_signals.py b/scripts/probe_signals.py new file mode 100644 index 0000000..c06f35e --- /dev/null +++ b/scripts/probe_signals.py @@ -0,0 +1,47 @@ +"""Print a short, secret-free snapshot of metrics, one log, and recent traces.""" + +import json +import urllib.parse +import urllib.request + + +def get(url): + with urllib.request.urlopen(url, timeout=10) as response: + return json.load(response) + + +def query(expr): + return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"] + + +names = get("http://127.0.0.1:9090/api/v1/label/__name__/values")["data"] +app = [name for name in names if name.startswith("http_") or name.startswith("messenger_")] +print("NAMES") +print("\n".join(app)) +for expr in ( + "messenger_jobs_total", + "messenger_queue_depth", + "messenger_db_up", + "messenger_worker_busy", + "messenger_worker_consumers", +): + print(expr, query(expr)) + +log_query = '{service_name="messenger-worker"}' +logs = get( + "http://127.0.0.1:3100/loki/api/v1/query_range?query=" + + urllib.parse.quote(log_query) + + "&limit=2" +) +streams = logs["data"]["result"] +print("LOG_STREAMS", len(streams)) +if streams: + print(streams[0]["values"][-1][1][:600]) + +tempo_query = '{resource.service.name="messenger-api"}' +traces = get( + "http://127.0.0.1:3200/api/search?q=" + urllib.parse.quote(tempo_query) + "&limit=5" +) +print("TRACES") +for item in traces.get("traces", []): + print(item.get("rootServiceName"), item.get("rootTraceName"), item.get("traceID"), item.get("durationMs")) diff --git a/scripts/probe_trace.py b/scripts/probe_trace.py new file mode 100644 index 0000000..b565e55 --- /dev/null +++ b/scripts/probe_trace.py @@ -0,0 +1,31 @@ +"""Summarize one Tempo trace by id. Prints span names and parent links only.""" + +import json +import sys +import urllib.request + +trace_id = sys.argv[1] +payload = json.load(urllib.request.urlopen(f"http://127.0.0.1:3200/api/traces/{trace_id}", timeout=10)) +batches = payload.get("batches") or payload.get("resourceSpans") or [] +# Tempo may return OTLP JSON under batches or the classic trace format. +if "trace" in payload: + spans = payload["trace"].get("spans") or [] + for span in spans: + print(span.get("name") or span.get("operationName"), span.get("spanID"), span.get("parentSpanID")) + raise SystemExit + +def walk(node): + if isinstance(node, dict): + if "name" in node and ("spanId" in node or "span_id" in node): + print( + node.get("name"), + "id=" + str(node.get("spanId") or node.get("span_id")), + "parent=" + str(node.get("parentSpanId") or node.get("parent_span_id") or "-"), + ) + for value in node.values(): + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + +walk(payload) diff --git a/scripts/render-s3-config.py b/scripts/render-s3-config.py new file mode 100644 index 0000000..5355548 --- /dev/null +++ b/scripts/render-s3-config.py @@ -0,0 +1,32 @@ +"""Write the local SeaweedFS identity file from .env. The file is gitignored.""" + +import json +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +env = {} +for line in (root / ".env").read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + env[key] = value + +out = root / "infra" / "seaweed" / "s3.json" +out.parent.mkdir(parents=True, exist_ok=True) +document = { + "identities": [ + { + "name": "app", + "credentials": [ + { + "accessKey": env["S3_ACCESS_KEY"], + "secretKey": env["S3_SECRET_KEY"], + } + ], + "actions": ["Admin", "Read", "List", "Write", "Tagging"], + } + ] +} +out.write_text(json.dumps(document), encoding="utf-8") +print("wrote infra/seaweed/s3.json") diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..4704277 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +wait_healthy() { + i=0 + while [ "$i" -lt 40 ]; do + ok=1 + for svc in "$@"; do + health=$(docker compose --env-file .env ps --format '{{.Service}} {{.Health}}' "$svc" | awk 'NR==1 {print $2}') + if [ "$health" != "healthy" ]; then + ok=0 + fi + done + if [ "$ok" -eq 1 ]; then + return 0 + fi + i=$((i + 1)) + sleep 1 + done + echo "timed out waiting for healthy: $*" >&2 + return 1 +} + +docker compose --env-file .env restart -t 5 postgres s3 rabbitmq +wait_healthy postgres s3 rabbitmq +docker compose --env-file .env restart -t 5 api-1 api-2 worker +wait_healthy api-1 api-2 worker +docker compose --env-file .env restart -t 2 nginx diff --git a/scripts/run_resilience_local.py b/scripts/run_resilience_local.py new file mode 100644 index 0000000..0380022 --- /dev/null +++ b/scripts/run_resilience_local.py @@ -0,0 +1,25 @@ +"""Run the public resilience test on Windows by executing the shell hook with Git Bash. + +The public test launches the hook directly. That works on Linux. This runner does not +change tests/resilience.py; it only prefixes the same script with bash. +""" + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +import resilience # noqa: E402 + +BASH = r"C:\Program Files\Git\bin\bash.exe" +_popen = resilience.subprocess.Popen + + +def popen(args, **kwargs): + if args and str(args[0]).endswith(".sh"): + args = [BASH, str(args[0])] + return _popen(args, **kwargs) + + +resilience.subprocess.Popen = popen +resilience.run(resilience.main) diff --git a/scripts/slo_experiment.py b/scripts/slo_experiment.py new file mode 100644 index 0000000..e90a0a0 --- /dev/null +++ b/scripts/slo_experiment.py @@ -0,0 +1,270 @@ +"""Track 2 measurements. Writes evidence/slo-track2.json without tokens or message text.""" + +import json +import sys +import time +import urllib.parse +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client # noqa: E402 + +FAULTS = ROOT / "faults" +OUT = ROOT / "evidence" / "slo-track2.json" + + +def get(url): + with urllib.request.urlopen(url, timeout=10) as response: + return json.load(response) + + +def prom(expr): + return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"] + + +def alert_states(): + found = {} + for item in prom('ALERTS{alertname=~"Api.*|QueueLagHigh|WorkerConsumersZero|AttachmentReadyLatencyHigh|DatabaseUnready"}'): + name = item["metric"].get("alertname") + found[name] = item["metric"].get("alertstate") + return found + + +def write_fault(name, value): + path = FAULTS / name + if value is None: + path.unlink(missing_ok=True) + else: + path.write_text(str(value), encoding="utf-8") + + +def clear_faults(): + for name in ("http_error_ratio", "db_slow_ms", "db_fail", "handler_slow_ms", "process_fail_remaining"): + write_fault(name, None) + + +def drive(client, actor, seconds, rps): + interval = 1.0 / rps + deadline = time.monotonic() + seconds + counts = {} + while time.monotonic() < deadline: + started = time.monotonic() + try: + result = client.call("GET", "/api/v1/chats", [200, 500], actor=actor) + status = result["status"] + except Exception: + status = "error" + counts[str(status)] = counts.get(str(status), 0) + 1 + sleep = interval - (time.monotonic() - started) + if sleep > 0: + time.sleep(sleep) + return counts + + +def wait_state(name, wanted, timeout): + started = time.monotonic() + last = {} + while time.monotonic() - started < timeout: + last = alert_states() + if last.get(name) == wanted or (wanted is None and name not in last): + return round(time.monotonic() - started, 1), last + time.sleep(5) + return None, last + + +def span_summary(trace_id): + payload = get(f"http://127.0.0.1:3200/api/traces/{trace_id}") + spans = [] + + def walk(node): + if isinstance(node, dict): + if "name" in node and "spanId" in node: + start = node.get("startTimeUnixNano") + end = node.get("endTimeUnixNano") + duration = None + try: + duration = round((int(end) - int(start)) / 1e6, 1) + except (TypeError, ValueError): + duration = None + spans.append( + { + "name": node.get("name"), + "span": node.get("spanId"), + "parent": node.get("parentSpanId") or "", + "ms": duration, + } + ) + for value in node.values(): + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(payload) + return spans + + +def latest_worker_trace(event): + query = '{service_name="messenger-worker"} |= "' + event + '"' + payload = get( + "http://127.0.0.1:3100/loki/api/v1/query_range?query=" + + urllib.parse.quote(query) + + "&limit=5" + ) + lines = [] + for stream in payload["data"]["result"]: + for _, line in stream["values"]: + try: + body = json.loads(line) + except ValueError: + continue + if body.get("event") == event: + lines.append(body) + if not lines: + return None + body = lines[-1] + return { + "event": body.get("event"), + "attempt": body.get("attempt"), + "trace_id": body.get("trace_id"), + "span_id": body.get("span_id"), + "service": body.get("service.name"), + } + + +def main(): + clear_faults() + report = {"started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} + client = Client("http://127.0.0.1:8080", timeout=8) + alice, _bob, _eve, chat = client.bootstrap() + print("bootstrap ok", flush=True) + + write_fault("process_fail_remaining", "1") + attachment, _content = client.attachment(alice, chat) + ready = client.wait_ready(alice, attachment, wait_seconds=40) + time.sleep(3) + log = latest_worker_trace("job.ready") + spans = [] + if log and log.get("trace_id"): + try: + spans = span_summary(log["trace_id"]) + except Exception as exc: + report["retry_trace_error"] = type(exc).__name__ + report["retry_upload"] = { + "final_status": ready["status"], + "log": log, + "consume_spans": sum(1 for span in spans if span["name"] == "attachment.process consume"), + "span_names": [span["name"] for span in spans], + "parents_linked": any(span["parent"] for span in spans), + } + print("retry", report["retry_upload"]["consume_spans"], report["retry_upload"]["log"], flush=True) + + print("warmup", flush=True) + report["warmup"] = drive(client, alice, 90, 2) + report["alerts_after_warmup"] = alert_states() + write_fault("http_error_ratio", "1") + print("short spike", flush=True) + report["short_spike"] = drive(client, alice, 10, 2) + time.sleep(35) + report["alerts_after_short_spike"] = alert_states() + clear_faults() + report["recovery_after_spike"] = drive(client, alice, 20, 2) + + write_fault("http_error_ratio", "1") + print("sustained", flush=True) + sustained_started = time.monotonic() + seen = None + counts = {} + deadline = time.monotonic() + 100 + interval = 0.5 + while time.monotonic() < deadline: + started = time.monotonic() + try: + status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"] + except Exception: + status = "error" + counts[str(status)] = counts.get(str(status), 0) + 1 + states = alert_states() + if seen is None and states.get("ApiErrorBudgetFastBurn") == "firing": + seen = round(time.monotonic() - sustained_started, 1) + report["fast_burn_detected_after_seconds"] = seen + report["alerts_when_fast_burn_fired"] = states + break + sleep = interval - (time.monotonic() - started) + if sleep > 0: + time.sleep(sleep) + report["sustained_counts_until_detection_or_timeout"] = counts + if seen is None: + report["fast_burn_detected_after_seconds"] = None + report["alerts_after_sustained"] = alert_states() + clear_faults() + print("resolve", flush=True) + resolve_started = time.monotonic() + resolved = None + resolve_deadline = time.monotonic() + 180 + while time.monotonic() < resolve_deadline: + client.call("GET", "/api/v1/chats", [200, 500], actor=alice) + if "ApiErrorBudgetFastBurn" not in alert_states(): + resolved = round(time.monotonic() - resolve_started, 1) + break + time.sleep(5) + report["fast_burn_resolved_after_seconds"] = resolved + report["alerts_after_resolve"] = alert_states() + + print("worker stop", flush=True) + import subprocess + + subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT) + client.attachment(alice, chat) + consumer_wait, consumer_state = wait_state("WorkerConsumersZero", "firing", 150) + queue_wait, queue_state = wait_state("QueueLagHigh", "firing", 150) + report["worker_stopped"] = { + "consumers_firing_after_seconds": consumer_wait, + "queue_lag_firing_after_seconds": queue_wait, + "alerts": queue_state or consumer_state, + } + subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT) + restored, restored_state = wait_state("WorkerConsumersZero", None, 120) + report["worker_restored"] = {"consumers_cleared_after_seconds": restored, "alerts": restored_state} + + print("latency split", flush=True) + write_fault("db_slow_ms", "800") + client.call("GET", "/api/v1/chats", [200, 500, 503], actor=alice) + time.sleep(3) + write_fault("db_slow_ms", None) + write_fault("handler_slow_ms", "800") + client.call("GET", "/api/v1/chats", [200, 500], actor=alice) + time.sleep(3) + write_fault("handler_slow_ms", None) + report["latency_note"] = "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span" + + print("tempo down", flush=True) + before = prom("otelcol_exporter_send_failed_spans") + subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT) + tempo_counts = drive(client, alice, 8, 2) + time.sleep(6) + after = prom("otelcol_exporter_send_failed_spans") + stats = subprocess.check_output( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"], + text=True, + ).strip() + subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT) + report["tempo_stopped"] = { + "api_results": tempo_counts, + "failed_spans_before": [item["value"][1] for item in before], + "failed_spans_after": [item["value"][1] for item in after], + "collector_memory": stats, + } + clear_faults() + report["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + print("wrote", OUT, flush=True) + + +if __name__ == "__main__": + try: + main() + finally: + clear_faults() diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..6b26290 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + api-1|api-2) ;; + *) echo "unexpected instance" >&2; exit 1 ;; +esac +docker compose --env-file .env stop -t 2 "$TARGET_INSTANCE_ID"