diff --git a/.env.example b/.env.example index 2b32655..d1a6c5e 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,5 @@ OTEL_SERVICE_NAME=messenger-api OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +OTEL_TRACES_SAMPLER_ARG=1.0 +POOL_MAX=10 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fe56092 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.sh text eol=lf +*.cfg text eol=lf +Dockerfile text eol=lf +*.sql text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1ab8004 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + PIP_DEFAULT_TIMEOUT=120 + +WORKDIR /app + +RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir --upgrade pip==25.0.1 \ + && pip install --no-cache-dir -r /app/requirements.txt + +COPY migrations /app/migrations +COPY src /app/src + +USER 10001 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=25s --retries=8 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" + +CMD ["python", "/app/src/main.py"] diff --git a/REPORT.md b/REPORT.md index b97ae42..49cde95 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,140 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Орлова Владислава, БПИ2301 +- Целевая оценка: 5. Трек 2, SLO и обнаружение инцидентов +- Commit/PR: ещё не публиковала +- Источники и помощь: задание и контракт этой лабораторной, рабочий стенд лабораторной 5, документация OpenTelemetry Collector, W3C Trace Context, Prometheus histograms и Grafana provisioning. При написании кода и отчёта пользовалась помощью ИИ. Числа ниже сняты с моего запуска 16 ноября 2026, я их не округляла ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine и Python 3.9+. У меня Docker Engine 26.1.4 и Python на Windows. Программы `make` в системе нет, поэтому команды из Makefile я запускала через `python -X utf8`. Флаг нужен, чтобы Windows не читала JSON в кодировке cp1251. Публичные `.sh` Windows не запускает как программу, их я запускала через WSL. + +```sh +python -X utf8 scripts/bootstrap.py +docker compose --env-file .env up -d --build --scale api=2 +python -X utf8 scripts/wait_ready.py +python -X utf8 scripts/check_contract.py +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +``` + +Проверки сохранности и остановки реплики: + +```sh +wsl -e bash -lc "cd '/mnt/c/Git/высоконагруженные приложения/Lab6' && python3 -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080" +wsl -e bash -lc "cd '/mnt/c/Git/высоконагруженные приложения/Lab6' && python3 -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080" +``` + +`bootstrap.py` один раз создаёт `secrets/` и `.env`. Пароли в консоль не печатаются. Повторный запуск уже созданные файлы не перезаписывает. В git эти файлы не входят. + +Остановить стенд, не удаляя данные: `docker compose --env-file .env stop`. Удалить учебные данные вместе с volume: `docker compose --env-file .env down -v`. + +Имя проекта Compose — `mtusi-lab6`. Соседняя папка лабораторной 5 из-за этого не пересекается со стендом. Наружу открыты `127.0.0.1:8080` у балансировщика, `127.0.0.1:3000` у Grafana и `127.0.0.1:9090` у Prometheus. OTLP наружу не публикую. + +Образы, которые реально поднимаются: + +| Роль | Образ | +| --- | --- | +| API и worker | свой Dockerfile, база `python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db` | +| Postgres | `postgres:16.10-bookworm@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74` | +| Redis | `redis:7.4.2-bookworm@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9` | +| RabbitMQ | `rabbitmq:3.13.7-alpine@sha256:d7af1c87c5f1eda13fcfca06db452bf3aeab6619fc3358b68535c0c02c4e52bc` | +| Балансировщик | `haproxy:2.8.14@sha256:2344d1efbf7a932c7a3d49b60748c28af81c70fd11fcadd983e2be1f0376a69d` | +| MinIO | `pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372` | +| OTel Collector | `otel/opentelemetry-collector-contrib:0.114.0@sha256:37fa87091cfaaec7234a27e4e395a40c31c2bfaea97a349a4afef6d9e9681197` | +| Prometheus | `prom/prometheus:v2.54.1@sha256:f6639335d34a77d9d9db382b92eeb7fc00934be8eae81dbc03b31cfe90411a94` | +| Loki | `grafana/loki:3.2.1@sha256:09a53b4a4ff81ffcd8f13886df19d33fac7a8d3aaf952e3c7e66cbade5b2fc31` | +| Tempo | `grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50` | +| Grafana | `grafana/grafana:11.2.2@sha256:d5133220d770aba5cb655147b619fa8770b90f41d8489a821d33b1cd34d16f89` | + +Сервер внутри образа MinIO отвечает `minio version RELEASE.2026-08-04T00-00-00Z`. Официальный `minio/minio` с Docker Hub больше не скачивается, а `quay.io/minio/minio` без логина отвечает `unauthorized`. Поэтому стенд закреплён на публичном образе `pgsty/minio` с digest. Это сервер MinIO с тем же протоколом S3. + +Grafana на учебном стенде: пользователь `admin`, пароль `labadmin`. Для просмотра dashboard достаточно анонимного Viewer. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Из лабораторной 5 я перенесла весь бизнес-стенд: две реплики API, HAProxy, Postgres, Redis-сессии, MinIO, RabbitMQ, worker, outbox, идемпотентность и старые текстовые правила. Чужой объект по-прежнему даёт 404. UUID в разном регистре — один идентификатор, в базе он в нижнем регистре. Миграции применяет одноразовый контейнер `migrate`. + +Новое в этой работе: API и worker отдают OTLP в Collector. Collector пишет metrics в Prometheus scrape endpoint `:8889`, traces в Tempo, logs в Loki. Grafana поднимается с provisioning datasources и dashboard `Messenger observability`. + +Соответствие сигналов `contracts/observability.md` и имён в Prometheus: + +| Сигнал | Имя в приложении | Имя в Prometheus | Единицы | +| --- | --- | --- | --- | +| HTTP requests | `messenger.http.requests` | `messenger_http_requests_total` | 1 | +| HTTP duration | `messenger.http.duration` | `messenger_http_duration_seconds` | секунды | +| Jobs processed | `messenger.jobs.processed` | `messenger_jobs_processed_total` | 1 | +| Job duration | `messenger.jobs.duration` | `messenger_jobs_duration_seconds` | секунды | +| Queue depth | `messenger.queue.depth` | `messenger_queue_depth` | 1 | +| Oldest job age | `messenger.queue.oldest_job_age` | `messenger_queue_oldest_job_age_seconds` | секунды | +| DB pool | `messenger.db.pool.used/waiting` | `messenger_db_pool_used_ratio` / `messenger_db_pool_waiting_ratio` | 1; суффикс `_ratio` ставит Prometheus exporter из-за unit=`1` | +| Worker saturation | `messenger.worker.busy/capacity` | `messenger_worker_busy_ratio` / `messenger_worker_capacity_ratio` | 1 | +| Time to ready | `messenger.attachment.time_to_ready` | `messenger_attachment_time_to_ready_seconds` | секунды | + +Labels метрик низкокардинальные: method, route template, status class, job type/outcome, queue name. `user_id` / `chat_id` / `message_id` в labels не использую. + +W3C trace context кладётся в поля `traceparent`/`tracestate` сообщения задания из `contracts/jobs.schema.json`. Worker открывает consumer span как child этого контекста. На прогоне upload trace `5a52413dfcd7d8afab436ca90e59877c` имел root `POST /api/v1/chats/{chat_id}/attachments` у `messenger-api` и продолжение обработки у worker в том же trace. + +Назначение хранилищ: + +- Prometheus — time series метрик и alert rules +- Loki — структурные логи с `trace_id`/`span_id` +- Tempo — трассы API и worker +- Grafana — запросы и dashboard, сама данные трёх сигналов не хранит +- Collector — приём OTLP, batch, memory limiter; не долговременная БД + +Collector: `memory_limiter` 256 MiB, batch, для Tempo/Loki очередь экспорта 256 и `max_elapsed_time=30s`. Metrics идут через pull Prometheus exporter, без бесконечной очереди в backend. Когда я останавливала Tempo/Loki/Prometheus, upload вернул 202, текстовое сообщение 201, ready остался 200, память Collector была `72.56MiB` до и `72.6MiB` после. + +Переменные из `.env.example`: `SESSION_STORE` в примере `postgres`, в compose у API стоит `redis`. `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318`, у worker отдельный `OTEL_SERVICE_NAME=messenger-worker`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Машина: Intel Core i5-10300H, 8 логических процессоров, 17008717824 байт памяти. Дата прогона 16 ноября 2026. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда и что я увидела | Файл | +| --- | --- | --- | +| Контракт на диске | `python -X utf8 scripts/check_contract.py` → PASS | — | +| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080` → `PASS: lab6 public API scenario...` | `evidence/smoke.txt` | +| Сохранность | `tests/resilience.py --mode persistence` через WSL, PASS. StartedAt сменился у postgres, redis, minio, rabbitmq, prometheus, loki, tempo и обеих API | `evidence/persistence.txt`, `evidence/persistence-before.txt`, `evidence/persistence-after.txt` | +| Остановка реплики | failover через WSL: 2 реплики до отказа, successful_writes=91, failed_attempts=1, first_recovery_seconds=1.307 | `evidence/failover.txt` | +| Негативные правила | `python -X utf8 student-tests/run_negative.py` → PASS negative | `evidence/negative.json` | +| Метрики/логи/traces | В Prometheus есть `messenger_http_*`, `messenger_jobs_*`, queue/pool/worker. В Loki есть `{service_name="messenger-api"} |= "http_request"` с `trace_id`. В Tempo upload и worker в одном trace | `evidence/observability.json`, `evidence/traces-queue.txt` | +| Инцидент worker stop | Возраст job вырос до 77.8 с, alert `GrowingQueueLag` firing через 99.1 с | `evidence/observability.json` | +| Инцидент DB stop | Прямые 503 в API, `HighHttpErrorRate` firing, `ErrorBudgetBurnFast` firing | `evidence/observability.json` | +| Telemetry backend down | API жив, RAM Collector не выросла | `evidence/observability.json` | +| Трек 2 burn-rate | Краткий всплеск 25 с: HighHttpErrorRate через 83.1 с, BurnFast не зажёгся. Длинная деградация 360 с: BurnSlow и BurnFast через 454.9 с; error_ratio_5m=1.0, error_ratio_30m≈0.601 | `evidence/observability.json` | + +Публичные файлы в `contracts/` и `tests/` я не меняла. + +p50/p95/p99 на dashboard считаются через `histogram_quantile` по bucket-метрикам, не через среднее. Запросы лежат в `infra/grafana/dashboards/messenger.json`. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### SLO -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +SLI API: доля HTTP 5xx на бизнес-маршрутах, health исключён. +SLO: error ratio ≤ 1% на учебном бюджете. +Окна burn-rate: fast = 14.4× на 5м и 30м, slow = 6× на 30м. Alert rules в `infra/prometheus/alerts.yml`, runbooks в `evidence/runbooks/`. + +Гипотеза трека 2: короткий всплеск должен зажечь обычный high-error alert и не обязан зажигать fast burn на двух окнах; длительная деградация должна зажечь slow burn. + +Что наблюдали: + +| Воздействие | Ожидание | Наблюдение | Восстановление | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| stop worker + upload | рост oldest age, GrowingQueueLag | age 31→78 с, firing через 99.1 с | start worker, файл доходит до ready | +| stop postgres + прямые запросы в API | 5xx, HighHttpErrorRate | 5xx counter 5→18, HighHttpErrorRate и BurnFast firing | start postgres, ready снова 200 | +| stop tempo/loki/prometheus | API жив, RAM Collector ограничена | upload 202, message 201, RAM 72.56→72.6 MiB | start backends | +| краткий postgres outage 25 с | HighHttpErrorRate, BurnFast может не зажечься | HighHttpErrorRate через 83.1 с, BurnFast = null | resolved после восстановления | +| длинный postgres outage 360 с | BurnSlow | BurnSlow и BurnFast через 454.9 с | resolved после восстановления | + +Ложное срабатывание BurnFast на коротком всплеске я не получила. Это ожидаемо: на окне 30м краткий отказ не держит 14.4× бюджета. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Стенд переживает остановку одной реплики API, рестарт хранилищ с сохранёнными volume, остановку worker на время приёма файла и краткую недоступность telemetry backends. Выключение компьютера и поломка диска данные не сохраняют. Postgres, Redis, MinIO, RabbitMQ, HAProxy и каждое telemetry-хранилище в стенде по одному экземпляру. + +Если обе реплики API красные по ready, клиент через HAProxy видит английскую HTML-страницу 503, а не JSON API. Поэтому для инцидента с Postgres я снимала бизнес-5xx прямым запросом внутрь контейнера API и честно написала об этом. + +Возраст очереди публикуют и API, и worker. Alert смотрит `max(...)`, иначе после остановки worker осталась бы устаревшая нулевая серия. + +Прямую загрузку по временной ссылке я не добавляла. Трек 1 про цену наблюдаемости не заявляю: отдельных измерений latency/CPU/RAM off/on у меня для этой работы нет, и я их не придумывала. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..2d857c5 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,425 @@ +name: mtusi-lab6 + +services: + postgres: + image: postgres:16.10-bookworm@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: + - postgres + - -c + - fsync=on + - -c + - synchronous_commit=on + - -c + - full_page_writes=on + - -c + - max_connections=80 + - -c + - shared_buffers=64MB + volumes: + - postgres-data:/var/lib/postgresql/data + secrets: + - postgres_password + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + cpus: "1.00" + mem_limit: 512m + restart: unless-stopped + networks: + - lab + + redis: + image: redis:7.4.2-bookworm@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9 + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --dir + - /data + - --save + - "60" + - "1" + - --bind + - 0.0.0.0 + - --protected-mode + - "no" + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 5s + cpus: "0.25" + mem_limit: 128m + restart: unless-stopped + networks: + - lab + + rabbitmq: + image: rabbitmq:3.13.7-alpine@sha256:d7af1c87c5f1eda13fcfca06db452bf3aeab6619fc3358b68535c0c02c4e52bc + hostname: rabbitmq + entrypoint: ["/bin/sh", "/init/rabbit-entry.sh"] + volumes: + - rabbitmq-data:/var/lib/rabbitmq + - ./scripts/rabbit-entry.sh:/init/rabbit-entry.sh:ro + secrets: + - rabbit_user + - rabbit_password + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 10s + retries: 12 + start_period: 180s + cpus: "1.00" + mem_limit: 1024m + restart: unless-stopped + networks: + - lab + + minio: + image: pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372 + command: ["server", "/data", "--console-address", "127.0.0.1:9001"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + volumes: + - minio-data:/data + secrets: + - minio_root_user + - minio_root_password + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 8s + cpus: "0.50" + mem_limit: 512m + restart: unless-stopped + networks: + - lab + + minio-init: + image: pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372 + entrypoint: ["/bin/sh", "/init/minio-init.sh"] + volumes: + - ./scripts/minio-init.sh:/init/minio-init.sh:ro + - ./infra/minio-api-policy.json:/init/policy.json:ro + secrets: + - minio_root_user + - minio_root_password + - s3_access_key + - s3_secret_key + depends_on: + minio: + condition: service_healthy + restart: "no" + networks: + - lab + + migrate: + build: . + image: mtusi-lab6-api:local + command: ["python", "/app/src/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + cpus: "0.50" + mem_limit: 256m + restart: "no" + networks: + - lab + + api: + build: . + image: mtusi-lab6-api:local + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: redis + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + POOL_MAX: ${POOL_MAX:-10} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: minio:9000 + S3_BUCKET: messenger-attachments + S3_SECURE: "false" + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-120} + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + SERVICE_VERSION: lab6 + secrets: + - database_url + - cache_url + - broker_url + - s3_access_key + - s3_secret_key + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + rabbitmq: + condition: service_healthy + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully + otel-collector: + condition: service_started + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 25s + init: true + stop_grace_period: 10s + cpus: "0.50" + mem_limit: 512m + deploy: + replicas: 2 + restart: unless-stopped + networks: + - lab + + worker: + build: . + image: mtusi-lab6-api:local + command: ["python", "/app/src/worker.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: minio:9000 + S3_BUCKET: messenger-attachments + S3_SECURE: "false" + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + SERVICE_VERSION: lab6 + secrets: + - database_url + - broker_url + - s3_access_key + - s3_secret_key + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully + otel-collector: + condition: service_started + healthcheck: + disable: true + init: true + stop_grace_period: 20s + cpus: "1.00" + mem_limit: 768m + restart: unless-stopped + networks: + - lab + + lb: + image: haproxy:2.8.14@sha256:2344d1efbf7a932c7a3d49b60748c28af81c70fd11fcadd983e2be1f0376a69d + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + api: + condition: service_healthy + cpus: "0.25" + mem_limit: 128m + restart: unless-stopped + networks: + - lab + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.114.0@sha256:37fa87091cfaaec7234a27e4e395a40c31c2bfaea97a349a4afef6d9e9681197 + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + tempo: + condition: service_started + cpus: "0.50" + mem_limit: 384m + restart: unless-stopped + networks: + - lab + + prometheus: + image: prom/prometheus:v2.54.1@sha256:f6639335d34a77d9d9db382b92eeb7fc00934be8eae81dbc03b31cfe90411a94 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=48h + - --web.enable-remote-write-receiver + - --web.enable-lifecycle + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - prometheus-data:/prometheus + ports: + - "127.0.0.1:9090:9090" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + cpus: "0.50" + mem_limit: 512m + restart: unless-stopped + networks: + - lab + + loki: + image: grafana/loki:3.2.1@sha256:09a53b4a4ff81ffcd8f13886df19d33fac7a8d3aaf952e3c7e66cbade5b2fc31 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki-data:/loki + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + cpus: "0.50" + mem_limit: 512m + restart: unless-stopped + networks: + - lab + + tempo: + image: grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo-data:/var/tempo + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + cpus: "0.50" + mem_limit: 512m + restart: unless-stopped + networks: + - lab + + grafana: + image: grafana/grafana:11.2.2@sha256:d5133220d770aba5cb655147b619fa8770b90f41d8489a821d33b1cd34d16f89 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: labadmin + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_SERVER_HTTP_PORT: "3000" + GF_PATHS_PROVISIONING: /etc/grafana/provisioning + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + ports: + - "127.0.0.1:3000:3000" + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + tempo: + condition: service_started + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 20s + cpus: "0.50" + mem_limit: 384m + restart: unless-stopped + networks: + - lab + +networks: + lab: + name: mtusi-lab6-net + +volumes: + postgres-data: + redis-data: + minio-data: + rabbitmq-data: + prometheus-data: + loki-data: + tempo-data: + grafana-data: + +secrets: + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + postgres_password: + file: ./secrets/postgres_password + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key + rabbit_user: + file: ./secrets/rabbit_user + rabbit_password: + file: ./secrets/rabbit_password + broker_url: + file: ./secrets/broker_url diff --git a/evidence/README.md b/evidence/README.md index 8df7001..d6df15e 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,13 @@ # Доказательства -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Здесь лежат обезличенные результаты прогонов лабораторной 6: -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +- `smoke.txt` — публичный smoke +- `persistence*.txt` — сохранность и StartedAt контейнеров +- `failover.txt` — остановка одной API-реплики +- `negative.json` — негативные проверки контракта +- `observability.json` — инциденты, alerts, burn-rate, telemetry down +- `traces-queue.txt` — связка upload→worker trace и GrowingQueueLag +- `runbooks/` — локальные runbook для alerts + +Секреты, токены и тела сообщений сюда не кладу. diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..f0ee7f9 Binary files /dev/null and b/evidence/failover.txt differ diff --git a/evidence/negative-run.txt b/evidence/negative-run.txt new file mode 100644 index 0000000..97eaeb4 Binary files /dev/null and b/evidence/negative-run.txt differ diff --git a/evidence/negative.json b/evidence/negative.json new file mode 100644 index 0000000..14da916 --- /dev/null +++ b/evidence/negative.json @@ -0,0 +1,74 @@ +{ + "command": "python -X utf8 student-tests/run_negative.py", + "result": "PASS negative", + "results": [ + { + "name": "chat_uuid_case", + "status": 200 + }, + { + "name": "blank_text", + "status": 400 + }, + { + "name": "bad_json", + "status": 400 + }, + { + "name": "short_password", + "status": 400 + }, + { + "name": "omitted_equals_empty", + "status": 201 + }, + { + "name": "same_key_other_body", + "status": 409 + }, + { + "name": "no_second_row", + "status": 200 + }, + { + "name": "same_key_other_chat", + "status": 201 + }, + { + "name": "foreign_chat", + "status": 404 + }, + { + "name": "bad_cursor", + "status": 400 + }, + { + "name": "slash_name", + "status": 400 + }, + { + "name": "huge_pixels", + "status": 413 + }, + { + "name": "bad_png", + "status": 415 + }, + { + "name": "type_mismatch", + "status": 415 + }, + { + "name": "oversize", + "status": 413 + }, + { + "name": "unknown_attachment", + "status": 404 + }, + { + "name": "no_token", + "status": 401 + } + ] +} diff --git a/evidence/observability.json b/evidence/observability.json new file mode 100644 index 0000000..4f0fe7b --- /dev/null +++ b/evidence/observability.json @@ -0,0 +1,64 @@ +{ + "started_at": "2026-11-16T17:09:51.151551+00:00", + "finished_at": "2026-11-16T17:53:00+00:00", + "base_url": "http://127.0.0.1:8080", + "scenarios": { + "trace_link": { + "ok": true, + "note": "W3C: consumer span живёт в том же trace, что и POST upload", + "sample_trace_id": "5a52413dfcd7d8afab436ca90e59877c", + "root_service": "messenger-api", + "root_name": "POST /api/v1/chats/{chat_id}/attachments", + "request_id_example": "4a726eff-62a4-4d13-a0f6-3a1a8c7c274e", + "loki_http_request_sample": { + "event": "http_request", + "route": "/api/v1/users", + "status": 201, + "trace_id": "411d51400fd602a598e0bfc09cad04ed", + "span_id": "46aecc4198246684" + } + }, + "worker_stop": { + "ok": true, + "attachment_id": "ee0819de-ef13-4d74-9841-4321054e2dff", + "fired_after_seconds": 99.1, + "age_samples": [ + {"t": 52.9, "age": 31.374413}, + {"t": 59.5, "age": 47.035699}, + {"t": 72.6, "age": 57.191765}, + {"t": 92.1, "age": 77.813526}, + {"t": 98.2, "age": 77.813526} + ], + "alert": "GrowingQueueLag firing" + }, + "db_stop": { + "ok": true, + "http_5xx_before": 5.0, + "http_5xx_after": 18.0, + "high_error_seen_after_seconds": 0.9, + "alerts_seen": ["HighHttpErrorRate firing", "ErrorBudgetBurnFast firing", "ErrorBudgetBurnSlow pending"], + "note": "Бизнес-5xx снимала прямым запросом в контейнер API, потому что HAProxy при красном ready отдаёт свою HTML 503" + }, + "telemetry_backend_down": { + "ok": true, + "upload_status": 202, + "ready_status": 200, + "message_status": 201, + "message_id": "76d951e4-1a0b-4be9-9217-22526922ce4c", + "otel_mem_before": "72.56MiB / 384MiB", + "otel_mem_after": "72.6MiB / 384MiB" + }, + "burn_rate_spike": { + "ok": true, + "brief_spike_seconds": 25, + "brief_high_error_fired_after_seconds": 83.1, + "brief_burn_fast_fired_after_seconds": null, + "long_degradation_seconds": 360, + "long_burn_slow_fired_after_seconds": 454.9, + "long_burn_fast_fired_after_seconds": 454.9, + "error_ratio_5m": 1.0, + "error_ratio_30m": 0.6013241420811717, + "note": "Короткий всплеск зажёг HighHttpErrorRate и не зажёг BurnFast. Длинная деградация зажгла BurnSlow и BurnFast." + } + } +} diff --git a/evidence/persistence-after.txt b/evidence/persistence-after.txt new file mode 100644 index 0000000..342ea71 --- /dev/null +++ b/evidence/persistence-after.txt @@ -0,0 +1,13 @@ +/mtusi-lab6-rabbitmq-1 2026-11-16T18:28:15.409788935Z +/mtusi-lab6-tempo-1 2026-11-16T18:28:15.402512463Z +/mtusi-lab6-loki-1 2026-11-16T18:28:15.558035972Z +/mtusi-lab6-prometheus-1 2026-11-16T18:28:15.558016399Z +/mtusi-lab6-worker-1 2026-11-16T18:27:16.239021708Z +/mtusi-lab6-api-1 2026-11-16T18:28:38.065212222Z +/mtusi-lab6-api-2 2026-11-16T18:28:37.997671947Z +/mtusi-lab6-lb-1 2026-11-16T17:36:12.433640676Z +/mtusi-lab6-grafana-1 2026-11-16T16:45:56.954366023Z +/mtusi-lab6-otel-collector-1 2026-11-16T17:19:04.876580907Z +/mtusi-lab6-minio-1 2026-11-16T18:28:02.310130635Z +/mtusi-lab6-postgres-1 2026-11-16T18:28:02.454244411Z +/mtusi-lab6-redis-1 2026-11-16T18:28:02.440606451Z diff --git a/evidence/persistence-before.txt b/evidence/persistence-before.txt new file mode 100644 index 0000000..2e94e7c --- /dev/null +++ b/evidence/persistence-before.txt @@ -0,0 +1,13 @@ +/mtusi-lab6-worker-1 2026-11-16T18:27:16.239021708Z +/mtusi-lab6-api-1 2026-11-16T18:27:16.110150646Z +/mtusi-lab6-api-2 2026-11-16T18:27:16.763621666Z +/mtusi-lab6-prometheus-1 2026-11-16T17:45:48.743893558Z +/mtusi-lab6-lb-1 2026-11-16T17:36:12.433640676Z +/mtusi-lab6-grafana-1 2026-11-16T16:45:56.954366023Z +/mtusi-lab6-otel-collector-1 2026-11-16T17:19:04.876580907Z +/mtusi-lab6-tempo-1 2026-11-16T17:18:58.629211805Z +/mtusi-lab6-rabbitmq-1 2026-11-16T16:45:56.111337038Z +/mtusi-lab6-loki-1 2026-11-16T17:18:57.765840011Z +/mtusi-lab6-minio-1 2026-11-16T16:43:54.21378518Z +/mtusi-lab6-postgres-1 2026-11-16T17:26:31.932858009Z +/mtusi-lab6-redis-1 2026-11-16T16:43:54.228473573Z diff --git a/evidence/persistence-containers.txt b/evidence/persistence-containers.txt new file mode 100644 index 0000000..d15fa1d Binary files /dev/null and b/evidence/persistence-containers.txt differ diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..c25ceba Binary files /dev/null and b/evidence/persistence.txt differ diff --git a/evidence/runbooks/burn-rate.md b/evidence/runbooks/burn-rate.md new file mode 100644 index 0000000..b74ae70 --- /dev/null +++ b/evidence/runbooks/burn-rate.md @@ -0,0 +1,14 @@ +# Runbook: ErrorBudgetBurnFast / ErrorBudgetBurnSlow + +SLO API: доля HTTP 5xx на бизнес-маршрутах ≤ 1% на скользящем окне 30 суток (учебный бюджет). +Burn-rate: + +- fast: 14.4x бюджета одновременно на окнах 5м и 30м → короткий сильный всплеск +- slow: 6x бюджета на окне 30м → длительная деградация + +Шаги: + +1. Сравни `messenger:http_error_ratio:5m` и `messenger:http_error_ratio:30m` в Prometheus. +2. Fast без slow обычно значит краткий всплеск; slow значит проблема тянется. +3. Дальше действуй как в `high-error-rate.md`: trace → log → зависимость. +4. Зафиксируй время firing и resolved. Ложные срабатывания возможны на очень малом RPS из‑за деления на малый знаменатель. diff --git a/evidence/runbooks/high-error-rate.md b/evidence/runbooks/high-error-rate.md new file mode 100644 index 0000000..cdf1b41 --- /dev/null +++ b/evidence/runbooks/high-error-rate.md @@ -0,0 +1,8 @@ +# Runbook: HighHttpErrorRate + +1. Открой dashboard `Messenger observability` в Grafana на `http://127.0.0.1:3000`. +2. Смотри панель «API error ratio» и «API latency». Если растут только 5xx, это не клиентские 4xx. +3. В Tempo найди свежие spans `messenger-api` со status ERROR. По `http.route` отдели бизнес-маршруты от `/health/*`. +4. По `trace_id` из лога Loki (`{service_name="messenger-api"} |= "http_request"`) открой связанный trace. +5. Если в spans много ошибок psycopg/timeout — смотри Postgres и панель DB pool. Если ошибки только на upload — смотри MinIO и worker. +6. После восстановления убедись, что alert перешёл в `resolved`, а error ratio снова ниже 5%. diff --git a/evidence/runbooks/queue-lag.md b/evidence/runbooks/queue-lag.md new file mode 100644 index 0000000..5f1bb05 --- /dev/null +++ b/evidence/runbooks/queue-lag.md @@ -0,0 +1,8 @@ +# Runbook: GrowingQueueLag + +1. Смотри панель «Oldest job age» и «Queue depth» для `attachments.process`. +2. Проверь контейнер worker: `docker compose --env-file .env ps worker`. +3. Если worker остановлен — это ожидаемый учебный отказ. Запусти worker и дождись уменьшения возраста job. +4. Если worker жив, открой Tempo spans `attachments.process consume` и ищи retry/failed. +5. В Loki `{service_name="messenger-worker"}` ищи `job_retry` / `job_failed`. +6. Alert должен уйти в `resolved`, когда oldest age снова ниже 30 с. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..7fff3da Binary files /dev/null and b/evidence/smoke.txt differ diff --git a/evidence/traces-queue.txt b/evidence/traces-queue.txt new file mode 100644 index 0000000..078a6f3 Binary files /dev/null and b/evidence/traces-queue.txt differ diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..c3c100a --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,279 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "API RPS", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[1m]))", + "legendFormat": "rps", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "fillOpacity": 10 } + } + } + }, + { + "title": "API error ratio (5xx / all business)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "messenger:http_error_ratio:5m", + "legendFormat": "5m error ratio", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { "drawStyle": "line", "fillOpacity": 10 } + } + } + }, + { + "title": "API latency p50/p95/p99", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "Jobs ready / failed / retry", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum by (job_outcome) (rate(messenger_jobs_processed_total[1m]))", + "legendFormat": "{{job_outcome}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { "drawStyle": "line", "fillOpacity": 10 } + } + } + }, + { + "title": "Job duration p50/p95/p99", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(messenger_jobs_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(messenger_jobs_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(messenger_jobs_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "Time to ready p50/p95/p99", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(messenger_attachment_time_to_ready_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(messenger_attachment_time_to_ready_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(messenger_attachment_time_to_ready_seconds_bucket[5m])) by (le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "Queue depth", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "messenger_queue_depth{messaging_destination_name=~\"attachments\\\\.process|attachments\\\\.retry\"}", + "legendFormat": "{{messaging_destination_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { "drawStyle": "line", "fillOpacity": 10 } + } + } + }, + { + "title": "Oldest job age", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "max(messenger_queue_oldest_job_age_seconds{messaging_destination_name=\"attachments.process\"})", + "legendFormat": "oldest age", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "fillOpacity": 10 } + } + } + }, + { + "title": "DB pool used / waiting", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "messenger_db_pool_used_ratio", + "legendFormat": "used {{service_name}}", + "refId": "A" + }, + { + "expr": "messenger_db_pool_waiting_ratio", + "legendFormat": "waiting {{service_name}}", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "Worker busy / capacity", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "messenger_worker_busy_ratio", + "legendFormat": "busy", + "refId": "A" + }, + { + "expr": "messenger_worker_capacity_ratio", + "legendFormat": "capacity", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "Process RSS", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "messenger_process_rss_bytes", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "API / worker logs", + "type": "logs", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 32 }, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [ + { + "expr": "{service_name=~\"messenger-api|messenger-worker\"}", + "refId": "A" + } + ] + } + ], + "schemaVersion": 39, + "tags": ["messenger", "lab6"], + "templating": { "list": [] }, + "time": { "from": "now-30m", "to": "now" }, + "timezone": "browser", + "title": "Messenger observability", + "uid": "messenger-lab6", + "version": 1 +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..7c36229 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..b89f255 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,45 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + uid: prometheus + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 15s + + - name: Loki + type: loki + access: proxy + uid: loki + url: http://loki:3100 + editable: false + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: '"trace_id":"([a-f0-9]+)"' + name: TraceID + url: "$${__value.raw}" + urlDisplayLabel: View Trace + + - name: Tempo + type: tempo + access: proxy + uid: tempo + url: http://tempo:3200 + editable: false + jsonData: + tracesToLogs: + datasourceUid: loki + filterByTraceID: true + filterBySpanID: false + mapTagNamesEnabled: false + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + lokiSearch: + datasourceUid: loki diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..94f7d80 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,47 @@ +global + maxconn 256 + tune.bufsize 65536 + tune.maxrewrite 8192 + +defaults + mode http + timeout connect 500ms + timeout client 60s + timeout server 60s + timeout http-request 60s + timeout http-keep-alive 2s + timeout queue 2s + timeout check 800ms + option redispatch + option dontlognull + retries 2 + retry-on conn-failure empty-response 502 503 504 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 1s + accepted_payload_size 8192 + +frontend web + bind *:8080 + default_backend api + +backend api + balance roundrobin + # Короткий JSON сообщения можно повторить. Загрузку файла не повторяем: + # повтор создал бы второй объект. Тело загрузки не буферизуется целиком. + acl upload method POST path_reg -i ^/api/v1/chats/[0-9a-fA-F-]+/attachments$ + http-request wait-for-body time 2s if METH_POST !upload + http-request disable-l7-retry if upload + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + server-template api 8 api:8080 check inter 500ms fall 2 rise 2 resolvers docker init-addr none resolve-prefer ipv4 on-marked-down shutdown-sessions diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..4d4798a --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,49 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /loki/chunks + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + ingestion_rate_mb: 16 + ingestion_burst_size_mb: 32 + allow_structured_metadata: true + volume_enabled: true + +ruler: + alertmanager_url: http://localhost:9093 + +analytics: + reporting_enabled: false diff --git a/infra/minio-api-policy.json b/infra/minio-api-policy.json new file mode 100644 index 0000000..cfef8ef --- /dev/null +++ b/infra/minio-api-policy.json @@ -0,0 +1,19 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..f5a6855 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -1,7 +1,9 @@ -# Конфигурация наблюдаемости — реализовать +# OpenTelemetry Collector -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. +Файл `collector.yaml` принимает OTLP HTTP/gRPC от API и worker, ограничивает память (`memory_limiter`), батчит экспорт и отправляет: -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. +- traces → Tempo (`otlphttp/tempo`) +- metrics → Prometheus remote write +- logs → Loki OTLP (`/otlp`) -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. +Очереди экспортеров ограничены (`queue_size: 256`). При недоступном backend retry ограничен `max_elapsed_time: 30s`, дальше сигнал отбрасывается. Это намеренно: приложение продолжает принимать сообщения, RAM Collector не растёт бесконечно. diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..d87ee40 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,80 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + send_batch_size: 256 + send_batch_max_size: 512 + timeout: 2s + resource: + attributes: + - key: deployment.environment.name + value: lab + action: upsert + +exporters: + otlphttp/tempo: + endpoint: http://tempo:4318 + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 256 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 30s + # Pull-модель: Collector держит только последние значения в памяти, без бесконечной очереди. + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + enable_open_metrics: true + otlphttp/loki: + endpoint: http://loki:3100/otlp + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 256 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 30s + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch, resource] + exporters: [otlphttp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch, resource] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch, resource] + exporters: [otlphttp/loki] + telemetry: + metrics: + level: basic + address: 0.0.0.0:8888 diff --git a/infra/prometheus/alerts.yml b/infra/prometheus/alerts.yml new file mode 100644 index 0000000..10d33a9 --- /dev/null +++ b/infra/prometheus/alerts.yml @@ -0,0 +1,88 @@ +groups: + - name: messenger-slo + rules: + - record: messenger:http_requests:rate5m + expr: sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[5m])) or vector(0) + - record: messenger:http_5xx:rate5m + expr: sum(rate(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[5m])) or vector(0) + - record: messenger:http_error_ratio:5m + expr: | + ( + messenger:http_5xx:rate5m + / + clamp_min(messenger:http_requests:rate5m, 0.001) + ) + - record: messenger:http_requests:rate30m + expr: sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[30m])) or vector(0) + - record: messenger:http_5xx:rate30m + expr: sum(rate(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[30m])) or vector(0) + - record: messenger:http_error_ratio:30m + expr: | + ( + messenger:http_5xx:rate30m + / + clamp_min(messenger:http_requests:rate30m, 0.001) + ) + - record: messenger:http_requests:rate1h + expr: sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[1h])) or vector(0) + - record: messenger:http_5xx:rate1h + expr: sum(rate(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[1h])) or vector(0) + - record: messenger:http_error_ratio:1h + expr: | + ( + messenger:http_5xx:rate1h + / + clamp_min(messenger:http_requests:rate1h, 0.001) + ) + + - name: messenger-alerts + rules: + - alert: HighHttpErrorRate + expr: messenger:http_error_ratio:5m > 0.05 + for: 1m + labels: + severity: page + annotations: + summary: "Доля HTTP 5xx выше 5% за 5 минут" + description: "Текущая доля {{ $value | humanizePercentage }}. Runbook: evidence/runbooks/high-error-rate.md" + runbook_url: "evidence/runbooks/high-error-rate.md" + + - alert: GrowingQueueLag + expr: max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"}) > 30 + for: 30s + labels: + severity: ticket + annotations: + summary: "Возраст старейшей job в attachments.process выше 30 с" + description: "Возраст {{ $value }} с. Runbook: evidence/runbooks/queue-lag.md" + runbook_url: "evidence/runbooks/queue-lag.md" + + - alert: ErrorBudgetBurnFast + expr: | + ( + messenger:http_error_ratio:5m > (14.4 * 0.01) + and + messenger:http_error_ratio:30m > (14.4 * 0.01) + ) + for: 2m + labels: + severity: page + burn_window: fast + annotations: + summary: "Быстрый burn rate ошибки SLO (14.4x на 5м и 30м)" + description: "Кратковременный всплеск 5xx. Runbook: evidence/runbooks/burn-rate.md" + runbook_url: "evidence/runbooks/burn-rate.md" + + - alert: ErrorBudgetBurnSlow + expr: | + ( + messenger:http_error_ratio:30m > (6 * 0.01) + ) + for: 5m + labels: + severity: ticket + burn_window: slow + annotations: + summary: "Медленный burn rate ошибки SLO (6x на 30м)" + description: "Длительная деградация. Runbook: evidence/runbooks/burn-rate.md" + runbook_url: "evidence/runbooks/burn-rate.md" diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..8f7a8a2 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,18 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alerts.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + - job_name: messenger-otel + metrics_path: /metrics + static_configs: + - targets: ["otel-collector:8889"] diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..fc4a97b --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,47 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + trace_idle_period: 10s + max_block_bytes: 1048576 + max_block_duration: 5m + +compactor: + compaction: + block_retention: 48h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + +overrides: + defaults: + metrics_generator: + processors: [] + +querier: + max_concurrent_queries: 20 + +query_frontend: + search: + max_duration: 1h diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 0000000..994fa1d --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,48 @@ +-- Пользователи, чаты, участники и сообщения. +-- Создание чата в приложении идёт одной транзакцией: строка chats и строки chat_members. +-- Индекс messages_history_idx совпадает с keyset-запросом истории: +-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id +-- LIMIT $4 + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_nonblank CHECK (btrim(display_name) <> ''), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT chats_title_nonblank CHECK (btrim(title) <> ''), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + ordinal smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_ordinal_nonneg CHECK (ordinal >= 0) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + body text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT messages_body_nonblank CHECK (btrim(body) <> ''), + CONSTRAINT messages_body_len CHECK (char_length(body) BETWEEN 1 AND 4000) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..d4e557b --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,19 @@ +-- Результат успешного POST сообщения. +-- Область ключа: пользователь + метод + путь чата + Idempotency-Key. +-- Строка пишется в той же транзакции, что и messages, поэтому 5xx +-- не оставляет «занятый» ключ. Другое тело при живом ключе — отказ +-- на уровне приложения (хеш не совпал), без второй строки messages. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + http_method text NOT NULL, + request_path text NOT NULL, + idem_key text NOT NULL, + request_hash text NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id), + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, http_method, request_path, idem_key), + CONSTRAINT idempotency_method_post CHECK (http_method = 'POST'), + CONSTRAINT idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_hash_len CHECK (char_length(request_hash) = 64) +); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..ebb52e3 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,42 @@ +-- Метаданные вложения. Байты лежат в приватном bucket, не в этой таблице. +-- Пустой текст допустим, если к сообщению прикреплено вложение. Пробелы по-прежнему нельзя. + +ALTER TABLE messages DROP CONSTRAINT messages_body_nonblank; +ALTER TABLE messages DROP CONSTRAINT messages_body_len; +ALTER TABLE messages + ADD CONSTRAINT messages_body_len CHECK (char_length(body) <= 4000); +ALTER TABLE messages + ADD CONSTRAINT messages_body_nonblank CHECK (body = '' OR btrim(body) <> ''); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL UNIQUE, + status text NOT NULL DEFAULT 'ready', + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type_chk CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size_chk CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha_chk CHECK (char_length(sha256) = 64), + CONSTRAINT attachments_status_chk CHECK (status = 'ready'), + CONSTRAINT attachments_key_chk CHECK (char_length(object_key) BETWEEN 1 AND 512) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + ordinal integer NOT NULL, + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, ordinal), + CONSTRAINT message_attachments_ordinal_chk CHECK (ordinal >= 0 AND ordinal < 10) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/004_jobs.sql b/migrations/004_jobs.sql new file mode 100644 index 0000000..f735350 --- /dev/null +++ b/migrations/004_jobs.sql @@ -0,0 +1,108 @@ +-- Задание на обработку, варианты и outbox. +-- Сообщение брокера не содержит байты и пароли: в outbox лежит только JSON задания. +-- Ключ эффекта — (attachment_id, pipeline_version, name). Повтор не создаёт вторую строку. + +ALTER TABLE attachments DROP CONSTRAINT attachments_status_chk; + +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; + +ALTER TABLE attachments ADD COLUMN error_code text; + +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; + +ALTER TABLE attachments ADD COLUMN staging_key text; + +ALTER TABLE attachments ADD COLUMN staging_deleted_at timestamptz; + +ALTER TABLE attachments ADD CONSTRAINT attachments_status_chk CHECK (status IN ('queued', 'processing', 'ready', 'failed')); + +ALTER TABLE attachments ADD CONSTRAINT attachments_pipeline_chk CHECK (pipeline_version >= 1); + +ALTER TABLE attachments ADD CONSTRAINT attachments_error_chk CHECK ( + (status = 'failed' AND error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted', 'processing_timeout')) + OR (status <> 'failed' AND error_code IS NULL) +); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + status text NOT NULL, + attempt integer NOT NULL, + max_attempts integer NOT NULL, + lease_owner text, + lease_until timestamptz, + next_attempt_at timestamptz, + error_code text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachment_jobs_status_chk CHECK (status IN ('queued', 'processing', 'ready', 'failed', 'superseded')), + CONSTRAINT attachment_jobs_attempt_chk CHECK (attempt >= 1 AND attempt <= max_attempts), + CONSTRAINT attachment_jobs_max_chk CHECK (max_attempts BETWEEN 1 AND 20), + CONSTRAINT attachment_jobs_version_chk CHECK (pipeline_version >= 1), + CONSTRAINT attachment_jobs_error_chk CHECK ( + error_code IS NULL OR error_code IN ( + 'invalid_image', 'processing_failed', 'retry_exhausted', 'processing_timeout', 'superseded' + ) + ), + CONSTRAINT attachment_jobs_one_version UNIQUE (attachment_id, pipeline_version) +); + +CREATE INDEX attachment_jobs_due_idx ON attachment_jobs (status, next_attempt_at); + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + name text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_name_chk CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_type_chk CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + CONSTRAINT attachment_variants_size_chk CHECK (size_bytes >= 1), + CONSTRAINT attachment_variants_dim_chk CHECK (width >= 1 AND height >= 1), + CONSTRAINT attachment_variants_sha_chk CHECK (sha256 ~ '^[a-f0-9]{64}$') +); + +CREATE TABLE outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id), + attempt integer NOT NULL, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + published_at timestamptz, + CONSTRAINT outbox_attempt_chk CHECK (attempt >= 1) +); + +CREATE INDEX outbox_pending_idx ON outbox (created_at) WHERE published_at IS NULL; + +CREATE TABLE job_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL, + attachment_id uuid NOT NULL, + pipeline_version integer NOT NULL, + kind text NOT NULL, + detail text, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT job_events_kind_chk CHECK (char_length(kind) BETWEEN 1 AND 40), + CONSTRAINT job_events_detail_chk CHECK (detail IS NULL OR char_length(detail) <= 64) +); + +CREATE TABLE fault_injections ( + id uuid PRIMARY KEY, + kind text NOT NULL, + attachment_id uuid, + armed boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fault_injections_kind_chk CHECK (kind IN ( + 'skip_inline_publish', 'crash_before_effect', 'crash_after_objects', + 'crash_after_commit', 'transient_once' + )) +); + +CREATE INDEX fault_injections_armed_idx ON fault_injections (kind, created_at) WHERE armed; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..be6bd23 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +pydantic==2.11.4 +psycopg[binary]==3.2.6 +psycopg-pool==3.2.4 +redis==5.2.1 +argon2-cffi==23.1.0 +minio==7.2.15 +pillow==11.1.0 +pika==1.3.2 +opentelemetry-api==1.27.0 +opentelemetry-sdk==1.27.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 +opentelemetry-semantic-conventions==0.48b0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 0000000..e1e4ba1 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Создать локальные .env и secrets для стенда. Пароль в консоль не печатается.""" +import secrets +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" +PASSWORD_FILE = SECRET_DIR / "postgres_password" +DATABASE_FILE = SECRET_DIR / "database_url" +CACHE_FILE = SECRET_DIR / "cache_url" +MINIO_USER_FILE = SECRET_DIR / "minio_root_user" +MINIO_PASSWORD_FILE = SECRET_DIR / "minio_root_password" +S3_ACCESS_FILE = SECRET_DIR / "s3_access_key" +S3_SECRET_FILE = SECRET_DIR / "s3_secret_key" +RABBIT_USER_FILE = SECRET_DIR / "rabbit_user" +RABBIT_PASSWORD_FILE = SECRET_DIR / "rabbit_password" +BROKER_FILE = SECRET_DIR / "broker_url" +ENV_FILE = ROOT / ".env" +ENV_EXAMPLE = ROOT / ".env.example" + + +def main(): + files = (PASSWORD_FILE, DATABASE_FILE, CACHE_FILE) + if any(path.exists() for path in files) and not all(path.exists() for path in files): + print("secrets/ заполнены не полностью, ничего не перезаписываю", file=sys.stderr) + return 1 + SECRET_DIR.mkdir(exist_ok=True) + if all(path.exists() for path in files): + print("secrets already exist") + else: + password = secrets.token_hex(24) + PASSWORD_FILE.write_text(password + "\n", encoding="utf-8") + DATABASE_FILE.write_text( + "postgresql://messenger:{}@postgres:5432/messenger\n".format(password), + encoding="utf-8", + ) + CACHE_FILE.write_text("redis://redis:6379/0\n", encoding="utf-8") + print("secrets created") + s3_files = (MINIO_USER_FILE, MINIO_PASSWORD_FILE, S3_ACCESS_FILE, S3_SECRET_FILE) + if any(path.exists() for path in s3_files) and not all(path.exists() for path in s3_files): + print("secrets/ для MinIO заполнены не полностью, ничего не перезаписываю", file=sys.stderr) + return 1 + if all(path.exists() for path in s3_files): + print("s3 secrets already exist") + else: + MINIO_USER_FILE.write_text("labroot", encoding="utf-8") + MINIO_PASSWORD_FILE.write_text(secrets.token_hex(24), encoding="utf-8") + S3_ACCESS_FILE.write_text("api" + secrets.token_hex(8), encoding="utf-8") + S3_SECRET_FILE.write_text(secrets.token_hex(24), encoding="utf-8") + print("s3 secrets created") + broker_files = (RABBIT_USER_FILE, RABBIT_PASSWORD_FILE, BROKER_FILE) + if any(path.exists() for path in broker_files) and not all(path.exists() for path in broker_files): + print("secrets/ для брокера заполнены не полностью, ничего не перезаписываю", file=sys.stderr) + return 1 + if all(path.exists() for path in broker_files): + print("broker secrets already exist") + else: + user = "lab" + password = secrets.token_hex(24) + RABBIT_USER_FILE.write_text(user, encoding="utf-8") + RABBIT_PASSWORD_FILE.write_text(password, encoding="utf-8") + BROKER_FILE.write_text( + "amqp://{}:{}@rabbitmq:5672/%2F\n".format(user, password), + encoding="utf-8", + ) + print("broker secrets created") + if ENV_FILE.exists(): + print(".env already exists") + else: + ENV_FILE.write_text(ENV_EXAMPLE.read_text(encoding="utf-8"), encoding="utf-8") + print(".env created") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/minio-init.sh b/scripts/minio-init.sh new file mode 100755 index 0000000..5685ff3 --- /dev/null +++ b/scripts/minio-init.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Идемпотентно создаёт приватный bucket и отдельного пользователя API. +# Корневой пароль остаётся только у этого контейнера и у MinIO. +set -eu +ROOT_USER=$(tr -d '\r\n' < /run/secrets/minio_root_user) +ROOT_PW=$(tr -d '\r\n' < /run/secrets/minio_root_password) +ACCESS=$(tr -d '\r\n' < /run/secrets/s3_access_key) +SECRET=$(tr -d '\r\n' < /run/secrets/s3_secret_key) + +attempt=0 +until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PW" >/dev/null; do + attempt=$((attempt + 1)) + if [ "$attempt" -gt 30 ]; then + echo "minio alias failed" >&2 + exit 1 + fi + sleep 1 +done + +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null + +if ! mc admin user info local "$ACCESS" >/dev/null 2>&1; then + mc admin user add local "$ACCESS" "$SECRET" >/dev/null +fi +mc admin policy detach local messenger-api --user "$ACCESS" >/dev/null 2>&1 || true +mc admin policy rm local messenger-api >/dev/null 2>&1 || true +mc admin policy create local messenger-api /init/policy.json >/dev/null +mc admin policy attach local messenger-api --user "$ACCESS" >/dev/null +echo "minio bucket ready" diff --git a/scripts/rabbit-entry.sh b/scripts/rabbit-entry.sh new file mode 100755 index 0000000..a81a07e --- /dev/null +++ b/scripts/rabbit-entry.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Пароль читается из secret и попадает только в файл конфигурации внутри контейнера. +set -eu +user=$(tr -d '\r\n' < /run/secrets/rabbit_user) +pass=$(tr -d '\r\n' < /run/secrets/rabbit_password) +mkdir -p /tmp/rabbit-conf +printf 'default_user = %s\ndefault_pass = %s\nvm_memory_high_watermark.absolute = 768MB\n' "$user" "$pass" > /tmp/rabbit-conf/rabbitmq.conf +chmod 644 /tmp/rabbit-conf/rabbitmq.conf +export RABBITMQ_CONFIG_FILE=/tmp/rabbit-conf/rabbitmq +exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..b6cf384 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Рестарт API и всех хранилищ со состоянием. Volume не удаляем, миграции заново не заливаем. +# Балансировщик без своего состояния не трогаем: он продолжает слушать тот же порт. +# Telemetry-хранилища тоже рестартуем: у них свои volume с историей. +docker compose --env-file .env restart postgres redis minio rabbitmq prometheus loki tempo +docker compose --env-file .env up -d --wait --no-deps postgres redis minio rabbitmq prometheus loki tempo +docker compose --env-file .env restart api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..0fcf468 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?}" + +found="" +for id in $(docker ps -q \ + --filter label=com.docker.compose.project=mtusi-lab6 \ + --filter label=com.docker.compose.service=api); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + short=$(printf %s "$id" | cut -c1-12) + if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then + found=$id + break + fi +done + +if [ -z "$found" ]; then + echo "api replica not found" >&2 + exit 1 +fi + +# SIGTERM, затем процесс остаётся остановленным (restart: unless-stopped). +docker stop --time 5 "$found" >/dev/null diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 0000000..dd556dc --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Ждать GET /health/ready через балансировщик. Без токенов и тел ошибок в выводе.""" +import os +import sys +import time +import urllib.error +import urllib.request + +URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + "/health/ready" + + +def main(): + deadline = time.monotonic() + 90 + last = "not-started" + while time.monotonic() < deadline: + try: + response = urllib.request.urlopen(URL, timeout=2) + if response.status == 200 and response.headers.get("X-Instance-Id"): + print("ready") + return 0 + last = str(response.status) + except (urllib.error.URLError, TimeoutError, OSError) as exc: + last = type(exc).__name__ + time.sleep(1) + print("not ready: " + last, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/blobs.py b/src/blobs.py new file mode 100644 index 0000000..1da6f09 --- /dev/null +++ b/src/blobs.py @@ -0,0 +1,228 @@ +"""Проверка PNG/JPEG и доступ к приватному bucket. Ключ объекта назначает сервер.""" +import io +import os +from datetime import datetime, timezone + +from minio import Minio +from minio.error import S3Error +from PIL import Image, UnidentifiedImageError +from PIL.Image import DecompressionBombError + +from errors import ApiError, DependencyDown + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") + +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + +_client = None + + +def read_secret(path): + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def client(): + global _client + if _client is None: + endpoint = os.environ.get("S3_ENDPOINT", "minio:9000").strip() + access = read_secret(os.environ["S3_ACCESS_KEY_FILE"]) + secret = read_secret(os.environ["S3_SECRET_KEY_FILE"]) + secure = os.environ.get("S3_SECURE", "false").strip().lower() == "true" + _client = Minio(endpoint, access_key=access, secret_key=secret, secure=secure) + return _client + + +def bucket_ready(): + try: + return client().bucket_exists(BUCKET) + except Exception: + return False + + +def put_bytes(key, data, content_type, created_at): + stamp = created_at.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + try: + client().put_object( + BUCKET, + key, + io.BytesIO(data), + length=len(data), + content_type=content_type, + metadata={"created-at": stamp}, + ) + except S3Error as exc: + raise DependencyDown() from exc + + +def delete_quiet(key): + try: + client().remove_object(BUCKET, key) + return True + except Exception: + return False + + +def open_object(key): + try: + return client().get_object(BUCKET, key) + except S3Error as exc: + code = getattr(exc, "code", "") + if code in ("NoSuchKey", "NoSuchBucket"): + return None + raise DependencyDown() from exc + + +def object_key(chat_id, attachment_id): + return "chats/{}/attachments/{}/original".format(chat_id, attachment_id) + + +def staging_object_key(attachment_id): + return "staging/{}/original".format(attachment_id) + + +def variant_object_key(attachment_id, pipeline_version, name): + return "ready/{}/v{}/{}".format(attachment_id, int(pipeline_version), name) + + +def header_size(data, declared): + if declared == "image/png": + return _png_size(data) + if declared == "image/jpeg": + return _jpeg_size(data) + return None + + +def read_bytes(key): + body = open_object(key) + if body is None: + return None + try: + return body.read() + finally: + body.close() + body.release_conn() + + +def declared_type(header): + if not header: + return None + media = header.split(";", 1)[0].strip().lower() + if media in ("image/png", "image/jpeg"): + return media + return None + + +def _png_size(data): + if len(data) < 24 or data[:8] != b"\x89PNG\r\n\x1a\n" or data[12:16] != b"IHDR": + return None + width = int.from_bytes(data[16:20], "big") + height = int.from_bytes(data[20:24], "big") + return width, height + + +def _jpeg_size(data): + if len(data) < 4 or data[:2] != b"\xff\xd8": + return None + index = 2 + limit = len(data) + while index + 8 < limit: + if data[index] != 0xFF: + return None + while index < limit and data[index] == 0xFF: + index += 1 + if index >= limit: + return None + marker = data[index] + index += 1 + if marker in (0xD8, 0xD9): + continue + if index + 2 > limit: + return None + segment = int.from_bytes(data[index:index + 2], "big") + if segment < 2 or index + segment > limit: + return None + if marker in (0xC0, 0xC1, 0xC2): + if segment < 7: + return None + height = int.from_bytes(data[index + 3:index + 5], "big") + width = int.from_bytes(data[index + 5:index + 7], "big") + return width, height + index += segment + return None + + +def _too_many_pixels(width, height): + if width <= 0 or height <= 0: + raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG") + if width > MAX_PIXELS or height > MAX_PIXELS or width * height > MAX_PIXELS: + raise ApiError(413, "payload_too_large", "В изображении больше 40 миллионов пикселей") + + +def inspect_image(data, declared): + if declared == "image/png": + size = _png_size(data) + elif declared == "image/jpeg": + size = _jpeg_size(data) + else: + size = None + detected = None + if data.startswith(b"\x89PNG\r\n\x1a\n"): + detected = "image/png" + elif data.startswith(b"\xff\xd8\xff"): + detected = "image/jpeg" + if size is None or detected != declared: + raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG") + _too_many_pixels(size[0], size[1]) + try: + with Image.open(io.BytesIO(data)) as image: + image.load() + except DecompressionBombError as exc: + raise ApiError(413, "payload_too_large", "В изображении больше 40 миллионов пикселей") from exc + except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc: + raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG") from exc + return declared + + +def list_objects(prefix=""): + found = [] + prefixes = (prefix,) if prefix else ("chats/", "staging/", "ready/") + try: + for one in prefixes: + found.extend(_list_prefix(one)) + except S3Error as exc: + raise DependencyDown() from exc + return found + + +def _list_prefix(prefix): + found = [] + try: + for item in client().list_objects(BUCKET, prefix=prefix, recursive=True): + if item.is_dir: + continue + stat = client().stat_object(BUCKET, item.object_name) + raw = None + if stat.metadata: + meta = {str(key).lower(): value for key, value in stat.metadata.items()} + raw = meta.get("x-amz-meta-created-at") or meta.get("created-at") + created = _parse_stamp(raw) if raw else None + if created is None and item.last_modified is not None: + created = item.last_modified + found.append({"key": item.object_name, "created_at": created}) + except S3Error as exc: + raise DependencyDown() from exc + return found + + +def _parse_stamp(value): + text = value.strip().replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 0000000..2a48d56 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,143 @@ +"""RabbitMQ: publisher confirm при публикации, consumer ack отдельно в worker.""" +import json +import os +from contextlib import contextmanager + +import pika +from opentelemetry.trace import SpanKind +from psycopg.types.json import Json + +from blobs import read_secret +from telemetry import start_span + +PROCESS_QUEUE = "attachments.process" +RETRY_QUEUE = "attachments.retry" +FAILED_QUEUE = "attachments.failed" + + +def broker_url(): + return read_secret(os.environ["BROKER_URL_FILE"]) + + +def _parameters(): + params = pika.URLParameters(broker_url()) + params.heartbeat = 60 + params.blocked_connection_timeout = 10 + params.socket_timeout = 10 + params.connection_attempts = 1 + return params + + +def ensure_topology(channel): + channel.queue_declare(queue=PROCESS_QUEUE, durable=True) + channel.queue_declare( + queue=RETRY_QUEUE, + durable=True, + arguments={ + "x-dead-letter-exchange": "", + "x-dead-letter-routing-key": PROCESS_QUEUE, + }, + ) + channel.queue_declare(queue=FAILED_QUEUE, durable=True) + + +@contextmanager +def broker_channel(): + connection = pika.BlockingConnection(_parameters()) + channel = connection.channel() + channel.confirm_delivery() + ensure_topology(channel) + try: + yield channel + finally: + try: + connection.close() + except Exception: + pass + + +def publish_raw(channel, queue, body, expiration_ms=None): + fields = { + "delivery_mode": 2, + "content_type": "application/json", + } + if expiration_ms is not None: + fields["expiration"] = str(int(expiration_ms)) + with start_span( + "attachments.publish", + kind=SpanKind.PRODUCER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": queue, + "messaging.operation": "publish", + }, + ): + channel.basic_publish( + exchange="", + routing_key=queue, + body=body, + properties=pika.BasicProperties(**fields), + mandatory=True, + ) + + +def publish_pending(conn, only_job_id=None): + published = 0 + while published < 20: + with conn.transaction(): + conn.execute("SET LOCAL statement_timeout = '20s'") + params = [] + sql = """ + SELECT o.id, o.payload + FROM outbox o + JOIN attachment_jobs j ON j.id = o.job_id + WHERE o.published_at IS NULL + AND (j.next_attempt_at IS NULL OR j.next_attempt_at <= now()) + """ + if only_job_id is not None: + sql += " AND o.job_id = %s" + params.append(only_job_id) + sql += """ + ORDER BY o.created_at + FOR UPDATE OF o SKIP LOCKED + LIMIT 1 + """ + row = conn.execute(sql, params).fetchone() + if row is None: + return published + body = json.dumps( + row["payload"], + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + with broker_channel() as channel: + publish_raw(channel, PROCESS_QUEUE, body) + conn.execute( + "UPDATE outbox SET published_at = now() WHERE id = %s AND published_at IS NULL", + (row["id"],), + ) + published += 1 + if only_job_id is not None: + return published + return published + + +def publish_delay(body, delay_ms): + with broker_channel() as channel: + publish_raw(channel, RETRY_QUEUE, body, expiration_ms=max(1000, int(delay_ms))) + + +def publish_failed(body): + with broker_channel() as channel: + publish_raw(channel, FAILED_QUEUE, body) + + +def queue_depth(queue): + with broker_channel() as channel: + declared = channel.queue_declare(queue=queue, passive=True) + return int(declared.method.message_count) + + +def dump_payload(payload): + return Json(payload) diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 0000000..341036d --- /dev/null +++ b/src/errors.py @@ -0,0 +1,32 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +class DependencyDown(Exception): + """Postgres или Redis не ответили. Наружу это 503 без имени хранилища.""" + + +class IdempotencyConflict(Exception): + """Тот же Idempotency-Key уже зафиксирован с другим телом.""" + + +class AttachmentRejected(Exception): + """Вложение есть, но прикрепить его к сообщению нельзя.""" + + +class AttachmentNotReady(Exception): + """Байты ещё нельзя отдать: queued, processing или failed.""" + + +def error_body(code, message, request_id): + return { + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + } diff --git a/src/faults.py b/src/faults.py new file mode 100644 index 0000000..b18de66 --- /dev/null +++ b/src/faults.py @@ -0,0 +1,49 @@ +"""Точечные сбои для проверки. Без строки в fault_injections код их не включает.""" +import uuid + +from psycopg.pq import TransactionStatus + + +def _idle(conn): + # Чтение выше могло открыть транзакцию. Её откат не теряет запись: + # сама запись начинается следующим transaction() и делает COMMIT. + if conn.info.transaction_status != TransactionStatus.IDLE: + conn.rollback() + + +def take_fault(conn, kind, attachment_id=None): + _idle(conn) + with conn.transaction(): + row = conn.execute( + """ + UPDATE fault_injections AS target + SET armed = false + WHERE target.id = ( + SELECT id + FROM fault_injections + WHERE armed + AND kind = %s + AND (attachment_id IS NULL OR attachment_id = %s) + ORDER BY created_at + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + RETURNING id + """, + (kind, attachment_id), + ).fetchone() + return row is not None + + +def arm_fault(conn, kind, attachment_id=None): + fault_id = uuid.uuid4() + _idle(conn) + with conn.transaction(): + conn.execute( + """ + INSERT INTO fault_injections (id, kind, attachment_id, armed) + VALUES (%s, %s, %s, true) + """, + (fault_id, kind, attachment_id), + ) + return str(fault_id) diff --git a/src/images.py b/src/images.py new file mode 100644 index 0000000..83a705e --- /dev/null +++ b/src/images.py @@ -0,0 +1,129 @@ +"""Варианты изображения. Оригинал не перекодируется.""" +import io + +from PIL import Image, ImageOps, UnidentifiedImageError +from PIL.Image import DecompressionBombError + +from blobs import MAX_PIXELS, header_size + +THUMB_BOX = 256 +CROP_SIZE = 128 +OPT_BOX = 1280 +JPEG_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 4 + +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +class PermanentJobError(Exception): + def __init__(self, code): + super().__init__(code) + self.code = code + + +class TransientJobError(Exception): + def __init__(self, code): + super().__init__(code) + self.code = code + + +def _rgb(image): + if image.mode == "RGBA" or image.mode == "LA" or (image.mode == "P" and "transparency" in image.info): + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + if image.mode == "RGB": + return image + return image.convert("RGB") + + +def load_rgb(data, declared): + size = header_size(data, declared) + detected = None + if data.startswith(b"\x89PNG\r\n\x1a\n"): + detected = "image/png" + elif data.startswith(b"\xff\xd8\xff"): + detected = "image/jpeg" + if size is None or detected != declared: + raise PermanentJobError("invalid_image") + width, height = size + if width <= 0 or height <= 0 or width > MAX_PIXELS or height > MAX_PIXELS or width * height > MAX_PIXELS: + raise PermanentJobError("invalid_image") + try: + with Image.open(io.BytesIO(data)) as image: + image.load() + oriented = ImageOps.exif_transpose(image) + rgb = _rgb(oriented) + rgb.load() + return rgb + except DecompressionBombError as exc: + raise PermanentJobError("invalid_image") from exc + except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc: + raise PermanentJobError("invalid_image") from exc + + +def fit_inside(image, box): + width, height = image.size + if width <= box and height <= box: + return image.copy() + if width >= height: + new_w = box + new_h = max(1, (height * box) // width) + else: + new_h = box + new_w = max(1, (width * box) // height) + return image.resize((new_w, new_h), Image.Resampling.LANCZOS) + + +def center_square(image, size): + width, height = image.size + side = min(width, height) + left = (width - side) // 2 + top = (height - side) // 2 + cropped = image.crop((left, top, left + side, top + side)) + if cropped.size != (size, size): + cropped = cropped.resize((size, size), Image.Resampling.LANCZOS) + return cropped + + +def _jpeg(image): + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=JPEG_QUALITY, optimize=True) + return { + "bytes": buffer.getvalue(), + "content_type": "image/jpeg", + "width": image.size[0], + "height": image.size[1], + } + + +def _webp(image): + buffer = io.BytesIO() + image.save(buffer, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD) + return { + "bytes": buffer.getvalue(), + "content_type": "image/webp", + "width": image.size[0], + "height": image.size[1], + } + + +def render(data, declared): + rgb = load_rgb(data, declared) + size = header_size(data, declared) + thumb = fit_inside(rgb, THUMB_BOX) + crop = center_square(rgb, CROP_SIZE) + optimized = fit_inside(rgb, OPT_BOX) + return { + "original": { + "bytes": data, + "content_type": declared, + "width": size[0], + "height": size[1], + }, + "thumbnail": _jpeg(thumb), + "crop": _jpeg(crop), + "optimized": _webp(optimized), + } diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..6186448 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,896 @@ +"""Состояния задания. Эффект один на (attachment_id, pipeline_version, вариант).""" +import hashlib +import json +import os +import random +import threading +import uuid +from datetime import datetime, timezone + +from psycopg.pq import TransactionStatus +from psycopg.types.json import Json + +from opentelemetry.trace import SpanKind, Status, StatusCode + +from blobs import delete_quiet, put_bytes, read_bytes, variant_object_key +from errors import DependencyDown +from broker import publish_delay, publish_failed, publish_raw, broker_channel, PROCESS_QUEUE +from faults import take_fault +from images import PermanentJobError, TransientJobError, render +from telemetry import ( + extract_context, + now_monotonic, + record_job, + record_time_to_ready, + start_span, + structured_log, +) + +REQUIRED = ("original", "thumbnail", "crop", "optimized") +PUBLIC_CODES = ("invalid_image", "processing_failed", "retry_exhausted", "processing_timeout") + + +class LostLease(Exception): + pass + + +def _idle(conn): + if conn.info.transaction_status != TransactionStatus.IDLE: + conn.rollback() + + +def max_attempts(): + raw = os.environ.get("JOB_MAX_ATTEMPTS", "5") + value = int(raw) + if not 1 <= value <= 20: + raise RuntimeError("JOB_MAX_ATTEMPTS") + return value + + +def lease_seconds(): + raw = os.environ.get("JOB_TIMEOUT_SECONDS", "60") + value = int(raw) + if value < 5: + raise RuntimeError("JOB_TIMEOUT_SECONDS") + return value + + +def retention_seconds(): + raw = os.environ.get("STAGING_RETENTION_SECONDS", "86400") + value = int(raw) + if value < 60: + raise RuntimeError("STAGING_RETENTION_SECONDS") + return value + + +def utc_now(): + return datetime.now(timezone.utc) + + +def dump_time(value): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + text = value.astimezone(timezone.utc).isoformat(timespec="microseconds") + if text.endswith("+00:00"): + text = text[:-6] + "Z" + return text + + +def backoff_seconds(failed_attempt): + base = min(30.0, 0.5 * (2 ** (failed_attempt - 1))) + return base + random.uniform(0, base * 0.25) + + +def job_message(job_id, attachment_id, pipeline_version, attempt, created_at, carrier=None): + if not isinstance(created_at, str): + created_at = dump_time(created_at) + payload = { + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": int(pipeline_version), + "attempt": int(attempt), + "created_at": created_at, + } + if carrier: + traceparent = carrier.get("traceparent") + tracestate = carrier.get("tracestate") + if isinstance(traceparent, str) and len(traceparent) == 55: + payload["traceparent"] = traceparent + if isinstance(tracestate, str) and 0 < len(tracestate) <= 512: + payload["tracestate"] = tracestate + return payload + + +def _uuid(value): + try: + return str(uuid.UUID(str(value))) + except (ValueError, AttributeError, TypeError): + return None + + +def _stamp(value): + if not isinstance(value, str) or not value.endswith("Z"): + return None + try: + datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError: + return None + return value + + +def parse_job(body): + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeError, ValueError, AttributeError): + return None + if not isinstance(payload, dict): + return None + allowed = { + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at", + "traceparent", + "tracestate", + } + if set(payload) - allowed: + return None + required = allowed - {"traceparent", "tracestate"} + if not required <= set(payload): + return None + if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1": + return None + job_id = _uuid(payload.get("job_id")) + attachment_id = _uuid(payload.get("attachment_id")) + created_at = _stamp(payload.get("created_at")) + version = payload.get("pipeline_version") + attempt = payload.get("attempt") + if job_id is None or attachment_id is None or created_at is None: + return None + if type(version) is not int or type(attempt) is not int: + return None + if version < 1 or attempt < 1: + return None + if "traceparent" in payload: + traceparent = payload["traceparent"] + if not isinstance(traceparent, str) or len(traceparent) != 55: + return None + if "tracestate" in payload: + tracestate = payload["tracestate"] + if not isinstance(tracestate, str) or len(tracestate) > 512: + return None + payload["job_id"] = job_id + payload["attachment_id"] = attachment_id + return payload + + +def _event(conn, job, kind, detail=None): + conn.execute( + """ + INSERT INTO job_events (job_id, attachment_id, pipeline_version, kind, detail) + VALUES (%s, %s, %s, %s, %s) + """, + (job["id"], job["attachment_id"], job["pipeline_version"], kind, detail), + ) + + +def _message_body(job, attempt, created_at): + payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], attempt, created_at) + return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def fail_job(conn, job, code): + if code not in PUBLIC_CODES and code != "superseded": + code = "processing_failed" + _idle(conn) + with conn.transaction(): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', error_code = %s, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = now() + WHERE id = %s AND status IN ('queued', 'processing') + """, + (code, job["id"]), + ) + if code in PUBLIC_CODES: + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = %s, updated_at = now() + WHERE id = %s AND pipeline_version = %s + AND status IN ('queued', 'processing') + """, + (code, job["attachment_id"], job["pipeline_version"]), + ) + _event(conn, job, "failed", code) + try: + publish_failed(_message_body(job, job["attempt"], job["created_at"])) + except Exception: + # Реестр failed уже в базе. Повторная доставка увидит failed и подтвердит сообщение. + return + + +def schedule_retry(conn, job, code, owner): + attempt = int(job["attempt"]) + if attempt >= int(job["max_attempts"]): + fail_job(conn, job, "retry_exhausted") + return + delay = backoff_seconds(attempt) + new_attempt = attempt + 1 + created = utc_now() + payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], new_attempt, created) + _idle(conn) + with conn.transaction(): + updated = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = %s, lease_owner = NULL, lease_until = NULL, + next_attempt_at = now() + make_interval(secs => %s), + error_code = NULL, updated_at = now() + WHERE id = %s AND status = 'processing' AND lease_owner = %s + RETURNING id + """, + (new_attempt, delay, job["id"], owner), + ).fetchone() + if updated is None: + return + conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status = 'processing' + """, + (job["attachment_id"], job["pipeline_version"]), + ) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (uuid.uuid4(), job["id"], new_attempt, Json(payload), created), + ) + _event(conn, job, "retry", code) + + +def release_expired(conn): + released = 0 + _idle(conn) + with conn.transaction(): + rows = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', lease_owner = NULL, lease_until = NULL, updated_at = now() + WHERE status = 'processing' AND lease_until < now() + RETURNING id, attachment_id, pipeline_version, attempt, created_at + """ + ).fetchall() + for row in rows: + payload = job_message(row["id"], row["attachment_id"], row["pipeline_version"], row["attempt"], utc_now()) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + SELECT %s, %s, %s, %s, now() + WHERE NOT EXISTS ( + SELECT 1 FROM outbox WHERE job_id = %s AND published_at IS NULL + ) + """, + (uuid.uuid4(), row["id"], row["attempt"], Json(payload), row["id"]), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status = 'processing' + """, + (row["attachment_id"], row["pipeline_version"]), + ) + _event(conn, row, "lease_expired", None) + released += 1 + return released + + +def cleanup_staging(conn): + removed = 0 + _idle(conn) + with conn.transaction(): + rows = conn.execute( + """ + SELECT a.id, a.staging_key + FROM attachments a + WHERE a.staging_key IS NOT NULL + AND a.staging_deleted_at IS NULL + AND a.status IN ('ready', 'failed') + AND NOT EXISTS ( + SELECT 1 FROM attachment_jobs j + WHERE j.attachment_id = a.id + AND j.status IN ('queued', 'processing') + ) + """ + ).fetchall() + for row in rows: + if delete_quiet(row["staging_key"]): + with conn.transaction(): + conn.execute( + """ + UPDATE attachments + SET staging_deleted_at = now() + WHERE id = %s AND staging_deleted_at IS NULL + """, + (row["id"],), + ) + removed += 1 + return removed + + +def _load_job(conn, job_id): + return conn.execute( + """ + SELECT id, attachment_id, pipeline_version, status, attempt, max_attempts, + lease_owner, lease_until, next_attempt_at, error_code, created_at + FROM attachment_jobs + WHERE id = %s + """, + (job_id,), + ).fetchone() + + +def _load_attachment(conn, attachment_id): + return conn.execute( + """ + SELECT id, content_type, sha256, object_key, staging_key, staging_deleted_at, + status, pipeline_version + FROM attachments + WHERE id = %s + """, + (attachment_id,), + ).fetchone() + + +def _acquire(conn, job, owner, seconds): + _idle(conn) + with conn.transaction(): + locked = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'processing', lease_owner = %s, + lease_until = now() + make_interval(secs => %s), + updated_at = now() + WHERE id = %s + AND status IN ('queued', 'processing') + AND attempt = %s + AND pipeline_version = %s + AND (lease_owner IS NULL OR lease_owner = %s OR lease_until < now()) + RETURNING id + """, + (owner, seconds, job["id"], job["attempt"], job["pipeline_version"], owner), + ).fetchone() + if locked is None: + return False + conn.execute( + """ + UPDATE attachments + SET status = 'processing', updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status IN ('queued', 'processing') + """, + (job["attachment_id"], job["pipeline_version"]), + ) + _event(conn, job, "leased", None) + return True + + +def _variants(conn, attachment_id, version): + rows = conn.execute( + """ + SELECT name, object_key, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + """, + (attachment_id, version), + ).fetchall() + return {row["name"]: row for row in rows} + + +def _source(conn, attachment): + if attachment["staging_key"] and attachment["staging_deleted_at"] is None: + data = read_bytes(attachment["staging_key"]) + if data: + return data + data = read_bytes(attachment["object_key"]) + if data: + return data + row = conn.execute( + """ + SELECT object_key + FROM attachment_variants + WHERE attachment_id = %s AND name = 'original' + ORDER BY pipeline_version DESC + LIMIT 1 + """, + (attachment["id"],), + ).fetchone() + if row is None: + raise TransientJobError("processing_failed") + data = read_bytes(row["object_key"]) + if not data: + raise TransientJobError("processing_failed") + return data + + +def _render_limited(data, declared, seconds): + budget = max(5, seconds - 5) + box = [] + errors = [] + + def run(): + try: + box.append(render(data, declared)) + except Exception as exc: + errors.append(exc) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(budget) + if thread.is_alive(): + raise TransientJobError("processing_timeout") + if errors: + exc = errors[0] + if isinstance(exc, (PermanentJobError, TransientJobError)): + raise exc + raise PermanentJobError("invalid_image") + return box[0] + + +def _beat(pool, job_id, owner, seconds, stop): + pause = max(2, seconds // 3) + while not stop.wait(pause): + try: + with pool.connection() as conn: + with conn.transaction(): + conn.execute( + """ + UPDATE attachment_jobs + SET lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE id = %s AND lease_owner = %s AND status = 'processing' + """, + (seconds, job_id, owner), + ) + except Exception: + return + + +def _supersede(conn, job): + _idle(conn) + with conn.transaction(): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'superseded', error_code = 'superseded', lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = now() + WHERE id = %s AND status IN ('queued', 'processing') + """, + (job["id"],), + ) + _event(conn, job, "superseded", None) + + +def _finalize(conn, job, owner, rendered): + _idle(conn) + with conn.transaction(): + current = conn.execute( + """ + SELECT lease_owner, status, pipeline_version + FROM attachment_jobs + WHERE id = %s + FOR UPDATE + """, + (job["id"],), + ).fetchone() + if current is None or current["lease_owner"] != owner or current["status"] != "processing": + raise LostLease() + attachment = conn.execute( + """ + SELECT pipeline_version, status + FROM attachments + WHERE id = %s + FOR UPDATE + """, + (job["attachment_id"],), + ).fetchone() + if attachment is None or int(attachment["pipeline_version"]) > int(job["pipeline_version"]): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'superseded', error_code = 'superseded', lease_owner = NULL, + lease_until = NULL, updated_at = now() + WHERE id = %s + """, + (job["id"],), + ) + _event(conn, job, "superseded", None) + return "superseded" + now = utc_now() + for name in REQUIRED: + item = rendered[name] + conn.execute( + """ + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, size_bytes, + width, height, sha256, object_key, created_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING + """, + ( + job["attachment_id"], + job["pipeline_version"], + name, + item["content_type"], + len(item["bytes"]), + item["width"], + item["height"], + item["sha256"], + item["object_key"], + now, + ), + ) + stored = conn.execute( + """ + SELECT count(*) AS n + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + AND name IN ('original', 'thumbnail', 'crop', 'optimized') + """, + (job["attachment_id"], job["pipeline_version"]), + ).fetchone() + if int(stored["n"]) < len(REQUIRED): + raise TransientJobError("processing_failed") + original = rendered["original"] + conn.execute( + """ + UPDATE attachments + SET status = 'ready', error_code = NULL, pipeline_version = %s, + object_key = %s, updated_at = %s + WHERE id = %s AND pipeline_version <= %s + """, + ( + job["pipeline_version"], + original["object_key"], + now, + job["attachment_id"], + job["pipeline_version"], + ), + ) + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', error_code = NULL, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = %s + WHERE id = %s AND lease_owner = %s + """, + (now, job["id"], owner), + ) + _event(conn, job, "ready", None) + return "ready" + + +def _drop_staging(conn, attachment): + _idle(conn) + with conn.transaction(): + active = conn.execute( + """ + SELECT count(*) AS n + FROM attachment_jobs + WHERE attachment_id = %s AND status IN ('queued', 'processing') + """, + (attachment["id"],), + ).fetchone() + if int(active["n"]) > 0 or not attachment["staging_key"]: + return + if delete_quiet(attachment["staging_key"]): + with conn.transaction(): + conn.execute( + """ + UPDATE attachments + SET staging_deleted_at = now() + WHERE id = %s AND staging_deleted_at IS NULL + """, + (attachment["id"],), + ) + + +def process_delivery(pool, body, owner): + payload = parse_job(body) + if payload is None: + return "ack" + carrier = {} + if "traceparent" in payload: + carrier["traceparent"] = payload["traceparent"] + if "tracestate" in payload: + carrier["tracestate"] = payload["tracestate"] + parent = extract_context(carrier) if carrier else None + started = now_monotonic() + with start_span( + "attachments.process consume", + kind=SpanKind.CONSUMER, + context=parent, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": PROCESS_QUEUE, + "messaging.operation": "process", + "job.type": "attachment.process.v1", + }, + ) as span: + return _process_delivery_body(pool, body, owner, payload, span, started) + + +def _process_delivery_body(pool, body, owner, payload, span, started): + import logging + + logger = logging.getLogger("worker") + with pool.connection() as conn: + job = _load_job(conn, payload["job_id"]) + if job is None: + return "ack" + if job["status"] in ("ready", "failed", "superseded"): + return "ack" + if int(payload["attempt"]) != int(job["attempt"]) or int(payload["pipeline_version"]) != int(job["pipeline_version"]): + return "ack" + if job["next_attempt_at"] is not None and job["next_attempt_at"] > utc_now(): + delay = (job["next_attempt_at"] - utc_now()).total_seconds() + publish_delay(body, int(delay * 1000)) + return "ack" + with pool.connection() as conn: + attachment = _load_attachment(conn, job["attachment_id"]) + if attachment is None: + fail_job(conn, job, "invalid_image") + record_job("failed", now_monotonic() - started) + return "ack" + if int(attachment["pipeline_version"]) > int(job["pipeline_version"]): + _supersede(conn, job) + record_job("superseded", now_monotonic() - started) + return "ack" + if not _acquire(conn, job, owner, lease_seconds()): + publish_delay(body, 2000) + return "ack" + structured_log(logger, "job_leased", job_id=str(job["id"]), attempt=int(job["attempt"])) + stop = threading.Event() + beater = threading.Thread( + target=_beat, + args=(pool, job["id"], owner, lease_seconds(), stop), + daemon=True, + ) + beater.start() + try: + with start_span("s3.read_source", kind=SpanKind.CLIENT): + with pool.connection() as conn: + if take_fault(conn, "crash_before_effect", job["attachment_id"]): + os_exit() + if take_fault(conn, "transient_once", job["attachment_id"]): + raise TransientJobError("processing_failed") + present = _variants(conn, job["attachment_id"], job["pipeline_version"]) + source = None if set(REQUIRED) <= set(present) else _source(conn, attachment) + if source is None: + rendered = _rendered_from_rows(conn, job, present) + if source is not None: + if hashlib.sha256(source).hexdigest() != attachment["sha256"]: + raise PermanentJobError("invalid_image") + with start_span("image.render", kind=SpanKind.INTERNAL): + rendered = _render_limited(source, attachment["content_type"], lease_seconds()) + with start_span("s3.write_variants", kind=SpanKind.CLIENT): + for name in REQUIRED: + item = rendered[name] + item["sha256"] = hashlib.sha256(item["bytes"]).hexdigest() + item["object_key"] = variant_object_key(job["attachment_id"], job["pipeline_version"], name) + if name in present: + continue + put_bytes(item["object_key"], item["bytes"], item["content_type"], utc_now()) + with pool.connection() as conn: + if take_fault(conn, "crash_after_objects", job["attachment_id"]): + os_exit() + with start_span("db.finalize_ready", kind=SpanKind.CLIENT): + with pool.connection() as conn: + _finalize(conn, job, owner, rendered) + with pool.connection() as conn: + if take_fault(conn, "crash_after_commit", job["attachment_id"]): + os_exit() + with pool.connection() as conn: + _drop_staging(conn, attachment) + duration = now_monotonic() - started + record_job("ready", duration) + if job.get("created_at") is not None: + created = job["created_at"] + if created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + record_time_to_ready((utc_now() - created).total_seconds()) + structured_log(logger, "job_ready", job_id=str(job["id"]), duration_ms=round(duration * 1000, 3)) + span.set_status(Status(StatusCode.OK)) + return "ack" + except PermanentJobError as exc: + with pool.connection() as conn: + fail_job(conn, job, exc.code) + record_job("failed", now_monotonic() - started) + span.set_status(Status(StatusCode.ERROR, exc.code)) + structured_log(logger, "job_failed", level=30, job_id=str(job["id"]), error_code=exc.code) + return "ack" + except LostLease: + return "ack" + except (TransientJobError, DependencyDown) as exc: + code = exc.code if isinstance(exc, TransientJobError) else "processing_failed" + with pool.connection() as conn: + schedule_retry(conn, job, code, owner) + record_job("retry", now_monotonic() - started) + span.set_status(Status(StatusCode.ERROR, code)) + structured_log(logger, "job_retry", level=30, job_id=str(job["id"]), error_code=code) + return "ack" + finally: + stop.set() + + +def os_exit(): + os._exit(75) + + +def _rendered_from_rows(conn, job, present): + # Повтор после уже записанного эффекта не перекодирует файлы. + rendered = {} + for name in REQUIRED: + row = present[name] + data = read_bytes(row["object_key"]) + if data is None: + raise TransientJobError("processing_failed") + meta = conn.execute( + """ + SELECT content_type, width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name = %s + """, + (job["attachment_id"], job["pipeline_version"], name), + ).fetchone() + rendered[name] = { + "bytes": data, + "content_type": meta["content_type"], + "width": int(meta["width"]), + "height": int(meta["height"]), + "sha256": meta["sha256"], + "object_key": meta["object_key"], + } + return rendered + + +def bump_pipeline(conn, attachment_id): + _idle(conn) + with conn.transaction(): + row = conn.execute( + "SELECT id, pipeline_version FROM attachments WHERE id = %s FOR UPDATE", + (attachment_id,), + ).fetchone() + if row is None: + return None + version = int(row["pipeline_version"]) + 1 + job_id = uuid.uuid4() + created = utc_now() + final_key = variant_object_key(attachment_id, version, "original") + conn.execute( + """ + UPDATE attachments + SET pipeline_version = %s, status = 'queued', error_code = NULL, + object_key = %s, updated_at = %s + WHERE id = %s + """, + (version, final_key, created, attachment_id), + ) + attempts = max_attempts() + conn.execute( + """ + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, status, attempt, max_attempts, created_at, updated_at + ) VALUES (%s, %s, %s, 'queued', 1, %s, %s, %s) + """, + (job_id, attachment_id, version, attempts, created, created), + ) + payload = job_message(job_id, attachment_id, version, 1, created) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, 1, %s, %s) + """, + (uuid.uuid4(), job_id, Json(payload), created), + ) + conn.execute( + """ + INSERT INTO job_events (job_id, attachment_id, pipeline_version, kind, detail) + VALUES (%s, %s, %s, 'version', %s) + """, + (job_id, attachment_id, version, str(version)), + ) + return {"job_id": str(job_id), "pipeline_version": version} + + +def replay_job(conn, job_id): + _idle(conn) + with conn.transaction(): + job = conn.execute( + """ + SELECT id, attachment_id, pipeline_version, status, attempt, max_attempts, created_at + FROM attachment_jobs + WHERE id = %s + FOR UPDATE + """, + (job_id,), + ).fetchone() + if job is None: + return {"result": "missing"} + if job["status"] == "ready": + return {"result": "already_ready"} + names = conn.execute( + """ + SELECT name FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + """, + (job["attachment_id"], job["pipeline_version"]), + ).fetchall() + have = {row["name"] for row in names} + if set(REQUIRED) <= have: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', error_code = NULL, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = now() + WHERE id = %s AND status <> 'ready' + """, + (job["id"],), + ) + return {"result": "already_done"} + if job["status"] != "failed": + return {"result": "not_failed"} + created = utc_now() + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = 1, error_code = NULL, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = %s + WHERE id = %s + """, + (created, job["id"]), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = %s + WHERE id = %s AND pipeline_version = %s AND status = 'failed' + """, + (created, job["attachment_id"], job["pipeline_version"]), + ) + payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], 1, created) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, 1, %s, %s) + """, + (uuid.uuid4(), job["id"], Json(payload), created), + ) + _event(conn, job, "replay", None) + return {"result": "queued"} + + +def republish(conn, job_id, attempt=None): + job = conn.execute( + """ + SELECT id, attachment_id, pipeline_version, attempt, created_at + FROM attachment_jobs + WHERE id = %s + """, + (job_id,), + ).fetchone() + if job is None: + return False + chosen = int(job["attempt"] if attempt is None else attempt) + body = _message_body(job, chosen, utc_now()) + with broker_channel() as channel: + publish_raw(channel, PROCESS_QUEUE, body) + return True diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..0028d8a --- /dev/null +++ b/src/main.py @@ -0,0 +1,519 @@ +import json +import logging +import os +import re +import time +import uuid +from contextlib import asynccontextmanager + +import anyio +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response, StreamingResponse +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from pydantic import ValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException + +from blobs import MAX_BYTES, declared_type, inspect_image, open_object +from errors import ApiError, AttachmentNotReady, AttachmentRejected, DependencyDown, IdempotencyConflict, error_body +from schemas import CreateChat, CreateMessage, CreateUser +from store import Store, instance_id +from telemetry import record_http, route_template, setup as setup_telemetry, structured_log + +LOGGER = logging.getLogger("messenger") +store = Store() +INSTANCE_ID = instance_id() + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +MAX_JSON_BYTES = 65536 + + +@asynccontextmanager +async def lifespan(_app): + setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-api")) + await anyio.to_thread.run_sync(store.startup) + yield + store.shutting_down = True + await anyio.to_thread.run_sync(store.shutdown) + + +app = FastAPI( + title="mtusi-messenger", + version="lab6", + lifespan=lifespan, + redirect_slashes=False, + docs_url="/docs", + redoc_url=None, + openapi_url="/openapi.json", +) + + +def request_id_of(request: Request): + rid = getattr(request.state, "request_id", None) + if rid: + return rid + return str(uuid.uuid4()) + + +def stamp(request: Request, response): + response.headers["X-Request-Id"] = request_id_of(request) + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +def json_ok(request: Request, payload, status_code=200): + return stamp(request, JSONResponse(payload, status_code=status_code)) + + +def json_error(request: Request, status, code, message, extra_headers=None): + headers = {"X-Request-Id": request_id_of(request), "X-Instance-Id": INSTANCE_ID} + if extra_headers: + headers.update(extra_headers) + return JSONResponse(error_body(code, message, request_id_of(request)), status_code=status, headers=headers) + + +@app.middleware("http") +async def wrap_request(request: Request, call_next): + incoming = request.headers.get("X-Request-Id", "") + if incoming and len(incoming) <= 200 and incoming.isprintable() and incoming.strip(): + request.state.request_id = incoming.strip() + else: + request.state.request_id = str(uuid.uuid4()) + if store.shutting_down and request.url.path != "/health/live": + return json_error(request, 503, "unavailable", "Сервис временно недоступен") + started = time.perf_counter() + route = route_template(request.url.path) + is_health = request.url.path in ("/health/live", "/health/ready") + if is_health: + response = await call_next(request) + return stamp(request, response) + tracer = trace.get_tracer("messenger") + with tracer.start_as_current_span( + f"{request.method} {route}", + kind=SpanKind.SERVER, + attributes={ + "http.request.method": request.method, + "http.route": route, + "service.instance.id": INSTANCE_ID, + }, + ) as span: + span.set_attribute("http.request_id", request.state.request_id) + try: + response = await call_next(request) + except Exception as exc: + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + raise + stamped = stamp(request, response) + span.set_attribute("http.response.status_code", stamped.status_code) + if stamped.status_code >= 500: + span.set_status(Status(StatusCode.ERROR)) + duration = time.perf_counter() - started + record_http(request.method, request.url.path, stamped.status_code, duration) + structured_log( + LOGGER, + "http_request", + method=request.method, + route=route, + status=stamped.status_code, + request_id=request.state.request_id, + duration_ms=round(duration * 1000, 3), + ) + return stamped + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return json_error(request, exc.status, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def request_validation_handler(request: Request, _exc: RequestValidationError): + return json_error(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(ValidationError) +async def pydantic_handler(request: Request, _exc: ValidationError): + return json_error(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return json_error(request, 404, "not_found", "Объект не найден") + return json_error(request, exc.status_code if exc.status_code < 500 else 500, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(Exception) +async def unexpected_handler(request: Request, exc: Exception): + if isinstance(exc, ApiError): + return json_error(request, exc.status, exc.code, exc.message, exc.headers) + LOGGER.error("unhandled %s", type(exc).__name__) + return json_error(request, 500, "internal_error", "Внутренняя ошибка") + + +def unavailable(): + raise ApiError(503, "unavailable", "Сервис временно недоступен") + + +def bearer_unauthorized(): + raise ApiError(401, "unauthorized", "Требуется авторизация", {"WWW-Authenticate": BEARER_CHALLENGE}) + + +def basic_unauthorized(): + raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": BASIC_CHALLENGE}) + + +def invalid(): + raise ApiError(400, "invalid_request", "Некорректный запрос") + + +async def call_store(func, *args): + try: + return await anyio.to_thread.run_sync(func, *args) + except DependencyDown: + unavailable() + except IdempotencyConflict: + raise ApiError(409, "conflict", "Ключ уже использован с другим телом") + except AttachmentRejected: + raise ApiError(409, "conflict", "Вложение нельзя прикрепить") + except AttachmentNotReady: + raise ApiError(409, "conflict", "Вложение недоступно") + + +def parse_basic(request: Request): + header = request.headers.get("Authorization", "") + if not header.lower().startswith("basic "): + basic_unauthorized() + raw = header.split(" ", 1)[1].strip() + try: + import base64 + + decoded = base64.b64decode(raw, validate=True).decode("utf-8") + except (ValueError, UnicodeError): + basic_unauthorized() + if ":" not in decoded: + basic_unauthorized() + username, password = decoded.split(":", 1) + return username, password + + +def parse_bearer(request: Request): + header = request.headers.get("Authorization", "") + # Basic на бизнес-методе не открывает сессию. X-User-Id не смотрим. + if not header.lower().startswith("bearer "): + bearer_unauthorized() + token = header.split(" ", 1)[1].strip() + if len(token) < 32 or len(token) > 512 or any(ch.isspace() for ch in token): + bearer_unauthorized() + return token + + +async def current_user(request: Request): + token = parse_bearer(request) + user = await call_store(store.user_for_token, token) + if user is None: + bearer_unauthorized() + request.state.user_id = user["id"] + request.state.token = token + return user + + +def parse_chat_id(chat_id): + try: + return str(uuid.UUID(chat_id)) + except ValueError: + invalid() + + +def parse_limit(raw): + if raw is None: + return 50 + if not isinstance(raw, str) or len(raw) > 3 or not raw.isdigit(): + invalid() + value = int(raw) + if value < 1 or value > 100: + invalid() + return value + + +async def read_json(request: Request): + ctype = request.headers.get("content-type", "") + if "application/json" not in ctype.lower(): + raise ApiError(415, "unsupported_media_type", "Нужен application/json") + length = request.headers.get("content-length") + if length is not None and (not length.isdigit() or int(length) > MAX_JSON_BYTES): + raise ApiError(413, "payload_too_large", "Слишком большое тело") + raw = await request.body() + if len(raw) > MAX_JSON_BYTES: + raise ApiError(413, "payload_too_large", "Слишком большое тело") + try: + payload = json.loads(raw) + except (ValueError, UnicodeError): + invalid() + if not isinstance(payload, dict): + invalid() + return payload + + +def parse_idempotency_key(request: Request): + key = request.headers.get("Idempotency-Key") + if key is None or not IDEMPOTENCY_RE.fullmatch(key): + invalid() + return key + + +@app.get("/health/live") +async def liveness(request: Request): + return json_ok(request, {"status": "ok"}) + + +@app.get("/health/ready") +async def readiness(request: Request): + if store.shutting_down: + raise ApiError(503, "unavailable", "Сервис временно недоступен") + ok = await call_store(store.ready) + if not ok: + unavailable() + return json_ok(request, {"status": "ok"}) + + +@app.post("/api/v1/users", status_code=201) +async def register_user(request: Request, body: CreateUser): + user = await call_store(store.create_user, body.username, body.display_name, body.password) + if user is None: + raise ApiError(409, "conflict", "Имя пользователя занято") + return json_ok(request, user, 201) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request): + username, password = parse_basic(request) + session = await call_store(store.login, username, password) + if session is None: + basic_unauthorized() + request.state.user_id = session["user"]["id"] + return json_ok(request, session, 201) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def revoke_session(request: Request): + user = await current_user(request) + request.state.user_id = user["id"] + removed = await call_store(store.revoke, request.state.token) + if not removed: + bearer_unauthorized() + return stamp(request, Response(status_code=204)) + + +@app.get("/api/v1/users/me") +async def me(request: Request): + user = await current_user(request) + return json_ok(request, user) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await current_user(request) + payload = await read_json(request) + try: + body = CreateChat.model_validate(payload) + except ValidationError: + invalid() + try: + chat = await call_store(store.create_chat, body.title, body.member_ids, user["id"]) + except ValueError: + invalid() + return json_ok(request, chat, 201) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await current_user(request) + limit = parse_limit(request.query_params.get("limit")) + cursor = request.query_params.get("cursor") + if cursor is not None and (not cursor or len(cursor) > 512): + invalid() + try: + items, next_cursor = await call_store(store.list_chats, user["id"], limit, cursor) + except ValueError: + invalid() + return json_ok(request, {"items": items, "next_cursor": next_cursor}) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(request: Request, chat_id: str): + user = await current_user(request) + chat = await call_store(store.get_chat, parse_chat_id(chat_id), user["id"]) + if chat is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, chat) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(request: Request, chat_id: str): + user = await current_user(request) + canonical = parse_chat_id(chat_id) + payload = await read_json(request) + key = parse_idempotency_key(request) + try: + body = CreateMessage.model_validate(payload) + except ValidationError: + invalid() + try: + message = await call_store( + store.add_message, canonical, user["id"], body.text, key, body.attachment_ids + ) + except ValueError: + invalid() + if message is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, message, 201) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(request: Request, chat_id: str): + user = await current_user(request) + canonical = parse_chat_id(chat_id) + limit = parse_limit(request.query_params.get("limit")) + cursor = request.query_params.get("cursor") + if cursor is not None and (not cursor or len(cursor) > 512): + invalid() + try: + result = await call_store(store.list_messages, canonical, user["id"], limit, cursor) + except ValueError: + invalid() + if result is None: + raise ApiError(404, "not_found", "Объект не найден") + items, next_cursor = result + return json_ok(request, {"items": items, "next_cursor": next_cursor}) + + +def check_filename(value): + if not isinstance(value, str) or not value or len(value) > 255: + invalid() + if "/" in value or "\\" in value or "\x00" in value: + invalid() + if any(ord(char) < 32 or ord(char) == 127 for char in value): + invalid() + return value + + +def parse_attachment_id(value): + try: + return str(uuid.UUID(value)) + except (TypeError, ValueError): + invalid() + + +async def read_upload(request: Request): + header = request.headers.get("content-length") + if header is not None: + try: + announced = int(header) + except ValueError: + invalid() + if announced > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 МиБ") + if announced < 1: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + chunks = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 МиБ") + chunks.append(chunk) + if total < 1: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + return b"".join(chunks) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(request: Request, chat_id: str): + user = await current_user(request) + canonical = parse_chat_id(chat_id) + member = await call_store(store.is_member, canonical, user["id"]) + if not member: + raise ApiError(404, "not_found", "Объект не найден") + filename = check_filename(request.query_params.get("filename")) + media = declared_type(request.headers.get("content-type")) + if media is None: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + data = await read_upload(request) + content_type = await anyio.to_thread.run_sync(inspect_image, data, media) + saved = await call_store( + store.save_attachment, canonical, user["id"], filename, content_type, data + ) + if saved is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, saved, 202) + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(request: Request, attachment_id: str): + user = await current_user(request) + canonical = parse_attachment_id(attachment_id) + found = await call_store(store.get_attachment, canonical, user["id"]) + if found is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, found) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(request: Request, attachment_id: str): + user = await current_user(request) + canonical = parse_attachment_id(attachment_id) + variant = request.query_params.get("variant") + if variant is None: + variant = "original" + if variant not in ("original", "thumbnail", "crop", "optimized"): + invalid() + opened = await call_store(store.open_attachment, canonical, user["id"], variant) + if opened is None: + raise ApiError(404, "not_found", "Объект не найден") + + def chunks(): + body = open_object(opened["object_key"]) + if body is None: + return + try: + for piece in body.stream(64 * 1024): + yield piece + finally: + body.close() + body.release_conn() + + probe = await anyio.to_thread.run_sync(open_object, opened["object_key"]) + if probe is None: + raise ApiError(404, "not_found", "Объект не найден") + probe.close() + probe.release_conn() + response = StreamingResponse(chunks(), media_type=opened["content_type"]) + response.headers["Content-Length"] = str(opened["size_bytes"]) + return stamp(request, response) + + +def setup_logging(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + + +if __name__ == "__main__": + import uvicorn + + setup_logging() + setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-api")) + host = os.environ.get("APP_HOST", "0.0.0.0") + port = int(os.environ.get("APP_PORT", "8080")) + uvicorn.run( + app, + host=host, + port=port, + timeout_graceful_shutdown=5, + timeout_keep_alive=2, + access_log=False, + log_config=None, + ) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..cc35b87 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,112 @@ +"""Однократные миграции. Запускается отдельным контейнером, не каждой репликой API.""" +import logging +import os +import sys +import time +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row + +LOGGER = logging.getLogger("migrate") +MIGRATION_LOCK = 421003 +CONNECT_TIMEOUT = 3 + + +def read_secret_file(path): + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def database_url(): + path = os.environ.get("DATABASE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + raise RuntimeError("DATABASE_URL_FILE") + return url + + +def split_sql(script): + parts = [] + buf = [] + for line in script.splitlines(): + if line.strip().startswith("--"): + continue + buf.append(line) + if line.strip().endswith(";"): + statement = "\n".join(buf).strip() + if statement.endswith(";"): + statement = statement[:-1].strip() + if statement: + parts.append(statement) + buf = [] + tail = "\n".join(buf).strip().rstrip(";").strip() + if tail: + parts.append(tail) + return parts + + +def wait_postgres(conninfo): + last = None + for _ in range(60): + try: + with psycopg.connect(conninfo, connect_timeout=CONNECT_TIMEOUT) as conn: + conn.execute("SELECT 1") + return + except psycopg.OperationalError as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("postgres not ready: %s", last) + raise SystemExit(1) + + +def apply(conninfo): + directory = Path(__file__).resolve().parents[1] / "migrations" + with psycopg.connect(conninfo, connect_timeout=CONNECT_TIMEOUT, row_factory=dict_row) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK,)) + conn.commit() + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + conn.commit() + for path in sorted(directory.glob("*.sql")): + version = path.stem + found = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (version,), + ).fetchone() + conn.rollback() + if found: + LOGGER.info("migration %s already applied", version) + continue + with conn.transaction(): + for statement in split_sql(path.read_text(encoding="utf-8")): + conn.execute(statement) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (version,), + ) + LOGGER.info("migration %s applied", version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK,)) + conn.commit() + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + conninfo = database_url() + wait_postgres(conninfo) + apply(conninfo) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/passwords.py b/src/passwords.py new file mode 100644 index 0000000..67c8daa --- /dev/null +++ b/src/passwords.py @@ -0,0 +1,26 @@ +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHash, VerificationError, VerifyMismatchError + +# Параметры ближе к учебному стенду, чем к максимальным: time=2, 19 MiB, Argon2id. +# Хеш всё равно медленный password KDF, не SHA-256 и не открытый текст. +hasher = PasswordHasher( + time_cost=2, + memory_cost=19456, + parallelism=1, + hash_len=32, + salt_len=16, +) + +# Чтобы неизвестный username не отвечал заметно быстрее известного. +DUMMY_HASH = hasher.hash("dummy-password-not-a-user") + + +def hash_password(password): + return hasher.hash(password) + + +def verify_password(password, encoded): + try: + return hasher.verify(encoded, password) + except (VerifyMismatchError, VerificationError, InvalidHash): + return False diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 0000000..acb822d --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,174 @@ +"""Сверка строк attachments и объектов bucket. + +Свежий объект моложе grace не удаляется: загрузка могла записать файл +раньше, чем строка в базе. Строка, на которую ссылается сообщение, тоже +не удаляется, даже если файла уже нет. По умолчанию только план, без удалений. +""" +import argparse +import json +import os +import sys +import time +from datetime import datetime, timezone + +import psycopg +from psycopg.rows import dict_row + +from blobs import BUCKET, delete_quiet, list_objects + + +def grace_seconds(): + raw = os.environ.get("RECONCILE_GRACE_SECONDS", "120") + value = int(raw) + if value < 1: + raise RuntimeError("RECONCILE_GRACE_SECONDS") + return value + + +def database_url(): + path = os.environ.get("DATABASE_URL_FILE") + if not path: + raise RuntimeError("DATABASE_URL_FILE") + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def aware(value): + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def plan(grace): + started = time.perf_counter() + objects = list_objects() + listed = time.perf_counter() + now = datetime.now(timezone.utc) + with psycopg.connect(database_url(), row_factory=dict_row) as conn: + rows = conn.execute( + """ + SELECT a.id, a.object_key, a.staging_key, a.created_at, a.status, + EXISTS ( + SELECT 1 FROM message_attachments m + WHERE m.attachment_id = a.id + ) AS in_use + FROM attachments a + """ + ).fetchall() + variant_rows = conn.execute("SELECT object_key FROM attachment_variants").fetchall() + by_key = {} + for row in rows: + by_key[row["object_key"]] = row + if row["staging_key"]: + by_key[row["staging_key"]] = row + for item in variant_rows: + by_key.setdefault(item["object_key"], {"status": "variant"}) + object_keys = {item["key"] for item in objects} + actions = [] + preserved = [] + for item in objects: + row = by_key.get(item["key"]) + if row is not None: + continue + created = aware(item["created_at"]) or now + age = (now - created).total_seconds() + entry = {"key": item["key"], "age_seconds": round(age, 3)} + if age < grace: + entry["reason"] = "fresh_object" + preserved.append(entry) + continue + entry["action"] = "delete_orphan_object" + actions.append(entry) + for row in rows: + if row["object_key"] in object_keys: + continue + created = aware(row["created_at"]) or now + age = (now - created).total_seconds() + entry = { + "attachment_id": str(row["id"]), + "key": row["object_key"], + "age_seconds": round(age, 3), + } + if row["status"] != "ready": + entry["reason"] = "not_ready" + preserved.append(entry) + continue + if row["in_use"]: + entry["reason"] = "missing_but_in_use" + preserved.append(entry) + continue + if age < grace: + entry["reason"] = "fresh_row" + preserved.append(entry) + continue + entry["action"] = "delete_orphan_row" + actions.append(entry) + return { + "bucket": BUCKET, + "grace_seconds": grace, + "objects": len(objects), + "rows": len(rows), + "list_seconds": round(listed - started, 6), + "plan_seconds": round(time.perf_counter() - started, 6), + "actions": actions, + "preserved": preserved, + } + + +def apply_plan(report): + deleted_objects = 0 + deleted_rows = 0 + with psycopg.connect(database_url(), row_factory=dict_row) as conn: + for action in report["actions"]: + if action["action"] == "delete_orphan_object": + if delete_quiet(action["key"]): + deleted_objects += 1 + elif action["action"] == "delete_orphan_row": + with conn.transaction(): + result = conn.execute( + """ + DELETE FROM attachments + WHERE id = %s + AND status = 'ready' + AND NOT EXISTS ( + SELECT 1 FROM message_attachments m + WHERE m.attachment_id = attachments.id + ) + AND NOT EXISTS ( + SELECT 1 FROM attachment_jobs j + WHERE j.attachment_id = attachments.id + ) + AND NOT EXISTS ( + SELECT 1 FROM attachment_variants v + WHERE v.attachment_id = attachments.id + ) + """, + (action["attachment_id"],), + ) + deleted_rows += result.rowcount + report["deleted_objects"] = deleted_objects + report["deleted_rows"] = deleted_rows + return report + + +def main(): + parser = argparse.ArgumentParser(description="Сверить вложения в базе и в bucket") + parser.add_argument("--apply", action="store_true", help="выполнить удаления; без флага только план") + parser.add_argument("--grace", type=int, default=None) + args = parser.parse_args() + grace = args.grace if args.grace is not None else grace_seconds() + if grace < 1: + print("grace должен быть не меньше 1 секунды", file=sys.stderr) + return 2 + report = plan(grace) + report["apply"] = bool(args.apply) + if args.apply: + apply_plan(report) + print(json.dumps(report, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 0000000..009f577 --- /dev/null +++ b/src/replay.py @@ -0,0 +1,24 @@ +"""Явный повтор failed-задания. Уже готовый эффект второй раз не исполняется.""" +import json +import sys + +import psycopg +from psycopg.rows import dict_row + +from blobs import read_secret +from jobs import replay_job + + +def main(): + if len(sys.argv) != 2: + print("usage: replay.py JOB_ID", file=sys.stderr) + return 2 + url = read_secret(__import__("os").environ["DATABASE_URL_FILE"]) + with psycopg.connect(url, row_factory=dict_row) as conn: + result = replay_job(conn, sys.argv[1]) + print(json.dumps(result, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/src/schemas.py b/src/schemas.py new file mode 100644 index 0000000..e11752d --- /dev/null +++ b/src/schemas.py @@ -0,0 +1,128 @@ +import re +import uuid + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +def _visible(value, limit): + if not isinstance(value, str) or not value.strip() or len(value) > limit: + raise ValueError("text") + return value + + +class CreateUser(StrictModel): + username: str + display_name: str + password: str + + @field_validator("username") + @classmethod + def username_ok(cls, value): + if not isinstance(value, str) or not USERNAME_RE.fullmatch(value): + raise ValueError("username") + return value + + @field_validator("display_name") + @classmethod + def display_name_ok(cls, value): + return _visible(value, 100) + + @field_validator("password") + @classmethod + def password_ok(cls, value): + if not isinstance(value, str) or not value.strip(): + raise ValueError("password") + # Длина в code points, не в байтах: bcrypt тут не подходит из-за усечения UTF-8. + if len(value) < 12 or len(value) > 128: + raise ValueError("password") + return value + + +class CreateChat(StrictModel): + title: str + member_ids: list + + @field_validator("title") + @classmethod + def title_ok(cls, value): + return _visible(value, 100) + + @field_validator("member_ids") + @classmethod + def member_ids_ok(cls, value): + if not isinstance(value, list) or not (1 <= len(value) <= 100): + raise ValueError("member_ids") + # Канонический UUID (нижний регистр). Иначе один и тот же id + # в разном регистре выглядит как два участника и упирается в PK. + normalized = [] + seen = set() + for item in value: + if not isinstance(item, str): + raise ValueError("member_ids") + try: + canonical = str(uuid.UUID(item)) + except ValueError as exc: + raise ValueError("member_ids") from exc + if canonical in seen: + raise ValueError("member_ids") + seen.add(canonical) + normalized.append(canonical) + return normalized + + +class CreateMessage(StrictModel): + text: str + attachment_ids: list = [] + + @model_validator(mode="before") + @classmethod + def fill_attachments(cls, value): + # Нет поля и явный [] — одно и то же тело для Idempotency-Key. + if not isinstance(value, dict): + return value + data = dict(value) + data.setdefault("attachment_ids", []) + return data + + @field_validator("text") + @classmethod + def text_ok(cls, value): + if not isinstance(value, str) or len(value) > 4000: + raise ValueError("text") + if value != "" and not value.strip(): + raise ValueError("text") + return value + + @field_validator("attachment_ids") + @classmethod + def attachments_ok(cls, value): + if not isinstance(value, list) or len(value) > 10: + raise ValueError("attachment_ids") + normalized = [] + seen = set() + for item in value: + if not isinstance(item, str): + raise ValueError("attachment_ids") + try: + canonical = str(uuid.UUID(item)) + except ValueError as exc: + raise ValueError("attachment_ids") from exc + if canonical in seen: + raise ValueError("attachment_ids") + seen.add(canonical) + normalized.append(canonical) + return normalized + + @model_validator(mode="after") + def text_or_file(self): + if self.text == "" and not self.attachment_ids: + raise ValueError("text") + if self.text and not self.text.strip(): + raise ValueError("text") + return self diff --git a/src/store.py b/src/store.py new file mode 100644 index 0000000..5f98d89 --- /dev/null +++ b/src/store.py @@ -0,0 +1,924 @@ +import base64 +import hashlib +import json +import logging +import os +import secrets +import socket +import time +import uuid +from datetime import datetime, timedelta, timezone + +import psycopg +import redis +from psycopg.errors import CheckViolation, ForeignKeyViolation, UniqueViolation +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +from errors import AttachmentNotReady, AttachmentRejected, DependencyDown, IdempotencyConflict +from passwords import DUMMY_HASH, hash_password, verify_password + +LOGGER = logging.getLogger("messenger") + +POOL_TIMEOUT_SECONDS = 3 +STATEMENT_TIMEOUT_MS = 5000 +LOCK_TIMEOUT_MS = 3000 +REDIS_MAX_CONNECTIONS = 16 +REDIS_SOCKET_TIMEOUT = 2 +SCHEMA_VERSION = "004_jobs" + +SQL_MESSAGES_FIRST = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE chat_id = %s +ORDER BY created_at, id +LIMIT %s +""" + +SQL_MESSAGES_AFTER = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE chat_id = %s + AND (created_at, id) > (%s, %s) +ORDER BY created_at, id +LIMIT %s +""" + +SQL_MESSAGE_BY_ID = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE id = %s +""" + + +def instance_id(): + # Docker без hostname: в контейнере это id контейнера, он не меняется, + # пока живёт этот контейнер. stop-one.sh находит контейнер по этому значению. + return socket.gethostname() + + +def pool_max(): + raw = os.environ.get("POOL_MAX", "10") + try: + value = int(raw) + except ValueError as exc: + raise RuntimeError("POOL_MAX") from exc + if not 1 <= value <= 32: + raise RuntimeError("POOL_MAX") + return value + + +def utc_now(): + return datetime.now(timezone.utc) + + +def dump_time(value): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + text = value.astimezone(timezone.utc).isoformat(timespec="microseconds") + if text.endswith("+00:00"): + text = text[:-6] + "Z" + return text + + +def parse_time(value): + if not isinstance(value, str) or not value.endswith("Z"): + raise ValueError("time") + return datetime.fromisoformat(value[:-1] + "+00:00") + + +def encode_cursor(scope, created_at, item_id): + raw = "{}|{}|{}".format(scope, created_at, item_id).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor, expected_scope): + if not isinstance(cursor, str) or not cursor or len(cursor) > 512: + raise ValueError("cursor") + pad = "=" * (-len(cursor) % 4) + try: + decoded = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + except (ValueError, UnicodeError) as exc: + raise ValueError("cursor") from exc + parts = decoded.split("|") + if len(parts) != 3: + raise ValueError("cursor") + scope, created_at, item_id = parts + if scope != expected_scope: + raise ValueError("cursor") + parse_time(created_at) + try: + uuid.UUID(item_id) + except ValueError as exc: + raise ValueError("cursor") from exc + return created_at, item_id + + +def session_key(token): + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return "sess:" + digest + + +def message_fingerprint(text, attachment_ids): + # Канонический JSON: порядок ключей не важен, пропущенный attachment_ids равен []. + canonical = json.dumps( + {"attachment_ids": list(attachment_ids), "text": text}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def advisory_key(user_id, path, idem_key): + raw = "\0".join((user_id, "POST", path, idem_key)).encode("utf-8") + digest = hashlib.sha256(raw).digest() + return int.from_bytes(digest[:8], "big", signed=True) + + +def chat_path(chat_id): + return "/api/v1/chats/" + chat_id + "/messages" + + +def public_user(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": dump_time(row["created_at"]), + } + + +def public_message(row, attachment_ids=None): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "attachment_ids": list(attachment_ids or []), + "created_at": dump_time(row["created_at"]), + } + + +def public_variant(row): + return { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "width": int(row["width"]), + "height": int(row["height"]), + "sha256": row["sha256"], + } + + +def public_attachment(row, variants=None): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "sha256": row["sha256"], + "status": row["status"], + "variants": [public_variant(item) for item in (variants or [])], + "error_code": row["error_code"], + "created_at": dump_time(row["created_at"]), + "updated_at": dump_time(row["updated_at"]), + } + + +def public_chat(row, member_ids=None): + members = member_ids if member_ids is not None else row["member_ids"] + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(item) for item in members], + "created_at": dump_time(row["created_at"]), + } + + +def read_secret_file(path): + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def _db_down(exc): + raise DependencyDown() from exc + + +class Store: + def __init__(self): + self.pool = None + self.redis = None + self.ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + self.idem_ttl = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) + self.pool_size = pool_max() + self.shutting_down = False + if self.ttl < 1 or self.idem_ttl < 1: + raise RuntimeError("TTL") + kind = os.environ.get("SESSION_STORE", "redis").strip().lower() + if kind != "redis": + raise RuntimeError("SESSION_STORE") + + def database_url(self): + path = os.environ.get("DATABASE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + raise RuntimeError("DATABASE_URL_FILE") + return url + + def redis_url(self): + path = os.environ.get("CACHE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("CACHE_URL", "").strip() + if not url: + raise RuntimeError("CACHE_URL_FILE") + return url + + def startup(self): + conninfo = self.database_url() + self._wait_schema(conninfo) + self.pool = ConnectionPool( + conninfo, + min_size=1, + max_size=self.pool_size, + timeout=POOL_TIMEOUT_SECONDS, + max_waiting=64, + max_lifetime=300, + kwargs={ + "connect_timeout": POOL_TIMEOUT_SECONDS, + "row_factory": dict_row, + "application_name": "mtusi-lab6-" + instance_id(), + "options": "-c statement_timeout={} -c lock_timeout={} -c idle_in_transaction_session_timeout=15000".format( + STATEMENT_TIMEOUT_MS, + LOCK_TIMEOUT_MS, + ), + }, + open=True, + ) + pool = redis.ConnectionPool.from_url( + self.redis_url(), + max_connections=REDIS_MAX_CONNECTIONS, + socket_timeout=REDIS_SOCKET_TIMEOUT, + socket_connect_timeout=REDIS_SOCKET_TIMEOUT, + health_check_interval=30, + decode_responses=True, + ) + self.redis = redis.Redis(connection_pool=pool) + self._wait_redis() + self._wait_bucket() + try: + from broker import broker_channel, publish_pending + + with broker_channel(): + pass + with self._connect() as conn: + publish_pending(conn) + except Exception as exc: + LOGGER.warning("outbox startup deferred %s", type(exc).__name__) + self._start_queue_metrics() + LOGGER.info("ready instance=%s ttl=%s pool_max=%s", instance_id(), self.ttl, self.pool_size) + + def _start_queue_metrics(self): + # Метрики возраста очереди должны жить и при остановленном worker. + import threading + + def loop(): + from broker import PROCESS_QUEUE, RETRY_QUEUE, queue_depth + from telemetry import observe_process, set_queue_stats + + while not self.shutting_down: + try: + age = 0.0 + with self._connect() as conn: + row = conn.execute( + """ + SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) AS age + FROM attachment_jobs + WHERE status IN ('queued', 'processing') + """ + ).fetchone() + if row is not None and row["age"] is not None: + age = float(row["age"]) + try: + depth = queue_depth(PROCESS_QUEUE) + except Exception: + depth = 0 + try: + retry_depth = queue_depth(RETRY_QUEUE) + except Exception: + retry_depth = 0 + set_queue_stats(PROCESS_QUEUE, depth, age) + set_queue_stats(RETRY_QUEUE, retry_depth, 0.0) + self.refresh_pool_stats() + observe_process() + except Exception as exc: + LOGGER.warning("queue metrics %s", type(exc).__name__) + time.sleep(5) + + threading.Thread(target=loop, name="queue-metrics", daemon=True).start() + + def shutdown(self): + self.shutting_down = True + if self.pool is not None: + self.pool.close() + if self.redis is not None: + self.redis.close() + + def ready(self): + if self.shutting_down or self.pool is None or self.redis is None: + return False + try: + with self.pool.connection(timeout=POOL_TIMEOUT_SECONDS) as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (SCHEMA_VERSION,), + ).fetchone() + if row is None: + return False + self.redis.ping() + from blobs import bucket_ready + + if not bucket_ready(): + return False + except Exception: + LOGGER.warning("ready check failed") + return False + return True + + def _wait_redis(self): + last = None + for _ in range(60): + try: + self.redis.ping() + return + except redis.RedisError as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("redis not ready: %s", last) + raise SystemExit(1) + + def _wait_schema(self, conninfo): + # Схему накатывает сервис migrate. Реплики только ждут её, не применяют сами. + last = None + for _ in range(90): + try: + with psycopg.connect(conninfo, connect_timeout=POOL_TIMEOUT_SECONDS, row_factory=dict_row) as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (SCHEMA_VERSION,), + ).fetchone() + if row is not None: + return + last = "migration missing" + except psycopg.Error as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("schema not ready: %s", last) + raise SystemExit(1) + + def _wait_bucket(self): + from blobs import bucket_ready + + for _ in range(60): + if bucket_ready(): + return + time.sleep(1) + LOGGER.error("object storage not ready") + raise SystemExit(1) + + def refresh_pool_stats(self): + if self.pool is None: + return + try: + from telemetry import set_pool_stats + + stats = self.pool.get_stats() + used = int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0)) + waiting = int(stats.get("requests_waiting", 0)) + set_pool_stats(max(0, used), waiting) + except Exception: + pass + + def _connect(self): + self.refresh_pool_stats() + try: + return self.pool.connection(timeout=POOL_TIMEOUT_SECONDS) + except PoolTimeout as exc: + _db_down(exc) + + def create_user(self, username, display_name, password): + password_hash = hash_password(password) + user_id = uuid.uuid4() + created = utc_now() + try: + with self._connect() as conn: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (username) DO NOTHING + RETURNING id, username, display_name, created_at + """, + (user_id, username, display_name, password_hash, created), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_user(row) + + def login(self, username, password): + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = %s + """, + (username,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + verify_password(password, DUMMY_HASH) + return None + if not verify_password(password, row["password_hash"]): + return None + return self._issue(public_user(row)) + + def _issue(self, user): + token = secrets.token_urlsafe(32) + expires = utc_now() + timedelta(seconds=self.ttl) + payload = json.dumps( + {"user_id": user["id"], "expires_at": dump_time(expires)}, + separators=(",", ":"), + ) + # pxat — абсолютное время. Рестарт Redis не сдвигает срок. + # В значении тоже лежит expires_at: даже если TTL ключа разъедется, сессия кончится вовремя. + try: + self.redis.set(session_key(token), payload, pxat=int(expires.timestamp() * 1000)) + except redis.RedisError as exc: + raise DependencyDown() from exc + return { + "token": token, + "token_type": "Bearer", + "expires_at": dump_time(expires), + "user": user, + } + + def user_for_token(self, token): + key = session_key(token) + try: + raw = self.redis.get(key) + except redis.RedisError as exc: + raise DependencyDown() from exc + if not raw: + return None + try: + payload = json.loads(raw) + expires = parse_time(payload["expires_at"]) + user_id = payload["user_id"] + uuid.UUID(user_id) + except (ValueError, KeyError, TypeError): + return None + if utc_now() >= expires: + try: + self.redis.delete(key) + except redis.RedisError as exc: + raise DependencyDown() from exc + return None + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = %s + """, + (user_id,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_user(row) + + def revoke(self, token): + try: + deleted = self.redis.delete(session_key(token)) + except redis.RedisError as exc: + raise DependencyDown() from exc + return deleted == 1 + + def create_chat(self, title, member_ids, caller_id): + try: + caller_uuid = uuid.UUID(str(caller_id)) + member_uuids = [] + seen = set() + for item in member_ids: + member_uuid = uuid.UUID(str(item)) + if member_uuid in seen: + raise ValueError("members") + seen.add(member_uuid) + member_uuids.append(member_uuid) + except ValueError as exc: + raise ValueError("members") from exc + if caller_uuid not in seen: + raise ValueError("members") + chat_id = uuid.uuid4() + created = utc_now() + canonical_members = [str(item) for item in member_uuids] + try: + with self._connect() as conn: + try: + with conn.transaction(): + conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat_id, title, created), + ) + for ordinal, member_id in enumerate(member_uuids): + conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, ordinal) + VALUES (%s, %s, %s) + """, + (chat_id, member_id, ordinal), + ) + row = conn.execute( + "SELECT id, title, created_at FROM chats WHERE id = %s", + (chat_id,), + ).fetchone() + except ForeignKeyViolation as exc: + raise ValueError("members") from exc + except UniqueViolation as exc: + raise ValueError("members") from exc + except psycopg.OperationalError as exc: + _db_down(exc) + return public_chat(row, canonical_members) + + def _member(self, conn, chat_id, user_id): + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + + def get_chat(self, chat_id, user_id): + try: + with self._connect() as conn: + if self._member(conn, chat_id, user_id) is None: + return None + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.ordinal) AS member_ids + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = %s + GROUP BY c.id + """, + (chat_id,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_chat(row) + + def list_chats(self, user_id, limit, cursor): + after = None + if cursor is not None: + created_at, item_id = decode_cursor(cursor, "chats:" + user_id) + after = (parse_time(created_at), uuid.UUID(item_id)) + sql = """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.ordinal) AS member_ids + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = %s + JOIN chat_members m ON m.chat_id = c.id + """ + params = [user_id] + if after is not None: + sql += " WHERE (c.created_at, c.id) > (%s, %s)" + params.extend(after) + sql += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT %s" + params.append(limit + 1) + try: + with self._connect() as conn: + rows = conn.execute(sql, params).fetchall() + except psycopg.OperationalError as exc: + _db_down(exc) + page = [public_chat(row) for row in rows[:limit]] + if len(rows) > limit and page: + last = page[-1] + return page, encode_cursor("chats:" + user_id, last["created_at"], last["id"]) + return page, None + + def _attachment_ids(self, conn, message_ids): + if not message_ids: + return {} + rows = conn.execute( + """ + SELECT message_id, attachment_id + FROM message_attachments + WHERE message_id = ANY(%s::uuid[]) + ORDER BY ordinal + """, + (list(message_ids),), + ).fetchall() + found = {} + for row in rows: + found.setdefault(str(row["message_id"]), []).append(str(row["attachment_id"])) + return found + + def is_member(self, chat_id, user_id): + try: + with self._connect() as conn: + return self._member(conn, chat_id, user_id) is not None + except psycopg.OperationalError as exc: + _db_down(exc) + + def _variants(self, conn, attachment_id, pipeline_version): + return conn.execute( + """ + SELECT name, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name <> 'original' + ORDER BY CASE name + WHEN 'thumbnail' THEN 1 + WHEN 'crop' THEN 2 + WHEN 'optimized' THEN 3 + ELSE 4 + END + """, + (attachment_id, pipeline_version), + ).fetchall() + + def save_attachment(self, chat_id, owner_id, filename, content_type, data): + from psycopg.types.json import Json + + from blobs import delete_quiet, put_bytes, staging_object_key, variant_object_key + from broker import publish_pending + from faults import take_fault + from jobs import job_message, max_attempts + from telemetry import inject_trace + + attachment_id = uuid.uuid4() + job_id = uuid.uuid4() + created = utc_now() + digest = hashlib.sha256(data).hexdigest() + staging = staging_object_key(attachment_id) + final_key = variant_object_key(attachment_id, 1, "original") + carrier = inject_trace({}) + payload = job_message(job_id, attachment_id, 1, 1, created, carrier=carrier) + attempts = max_attempts() + put_bytes(staging, data, content_type, created) + try: + with self._connect() as conn: + with conn.transaction(): + if self._member(conn, chat_id, owner_id) is None: + raise LookupError("chat") + row = conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, object_key, staging_key, status, + pipeline_version, error_code, created_at, updated_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'queued', 1, NULL, %s, %s) + RETURNING id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, status, error_code, created_at, updated_at + """, + ( + attachment_id, + chat_id, + owner_id, + filename, + content_type, + len(data), + digest, + final_key, + staging, + created, + created, + ), + ).fetchone() + conn.execute( + """ + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, status, attempt, + max_attempts, created_at, updated_at + ) + VALUES (%s, %s, 1, 'queued', 1, %s, %s, %s) + """, + (job_id, attachment_id, attempts, created, created), + ) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, 1, %s, %s) + """, + (uuid.uuid4(), job_id, Json(payload), created), + ) + except (LookupError, ForeignKeyViolation): + delete_quiet(staging) + return None + except psycopg.OperationalError as exc: + delete_quiet(staging) + _db_down(exc) + except Exception: + delete_quiet(staging) + raise + try: + with self._connect() as conn: + skipped = take_fault(conn, "skip_inline_publish", attachment_id) + if not skipped: + with self._connect() as conn: + publish_pending(conn, only_job_id=job_id) + except Exception as exc: + LOGGER.warning("publish deferred %s", type(exc).__name__) + return public_attachment(row, []) + + def get_attachment(self, attachment_id, user_id): + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.status, a.error_code, a.pipeline_version, + a.created_at, a.updated_at + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s + WHERE a.id = %s + """, + (user_id, attachment_id), + ).fetchone() + variants = [] + if row is not None and row["status"] == "ready": + variants = self._variants(conn, row["id"], row["pipeline_version"]) + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_attachment(row, variants) + + def open_attachment(self, attachment_id, user_id, variant): + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT a.object_key, a.content_type, a.size_bytes, a.status, a.pipeline_version + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s + WHERE a.id = %s + """, + (user_id, attachment_id), + ).fetchone() + if row is None: + return None + if row["status"] != "ready": + raise AttachmentNotReady() + if variant == "original": + return { + "object_key": row["object_key"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + } + found = conn.execute( + """ + SELECT object_key, content_type, size_bytes + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name = %s + """, + (attachment_id, row["pipeline_version"], variant), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if found is None: + return None + return { + "object_key": found["object_key"], + "content_type": found["content_type"], + "size_bytes": int(found["size_bytes"]), + } + + def add_message(self, chat_id, sender_id, text, idem_key, attachment_ids): + # Один ключ сериализуется между репликами через advisory lock в той же + # транзакции, где INSERT сообщения. Откат не оставляет ключ занятым. + path = chat_path(chat_id) + digest = message_fingerprint(text, attachment_ids) + lock_key = advisory_key(sender_id, path, idem_key) + message_id = uuid.uuid4() + try: + with self._connect() as conn: + with conn.transaction(): + # Иначе lock_timeout сессии (3 с) оборвёт очередь одинаковых ключей. + conn.execute("SET LOCAL lock_timeout = '15s'") + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,)) + if self._member(conn, chat_id, sender_id) is None: + return None + existing = conn.execute( + """ + SELECT request_hash, message_id, + created_at >= now() - make_interval(secs => %s) AS fresh + FROM idempotency_keys + WHERE user_id = %s AND http_method = 'POST' + AND request_path = %s AND idem_key = %s + """, + (self.idem_ttl, sender_id, path, idem_key), + ).fetchone() + if existing is not None and existing["fresh"]: + if existing["request_hash"] != digest: + raise IdempotencyConflict() + row = conn.execute(SQL_MESSAGE_BY_ID, (existing["message_id"],)).fetchone() + if row is None: + raise RuntimeError("idempotency target missing") + linked = self._attachment_ids(conn, [row["id"]]).get(str(row["id"]), []) + return public_message(row, linked) + if existing is not None: + conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = %s AND http_method = 'POST' + AND request_path = %s AND idem_key = %s + """, + (sender_id, path, idem_key), + ) + if attachment_ids: + found = conn.execute( + """ + SELECT id, chat_id, status + FROM attachments + WHERE id = ANY(%s::uuid[]) + """, + (list(attachment_ids),), + ).fetchall() + by_id = {str(item["id"]): item for item in found} + for attachment_id in attachment_ids: + item = by_id.get(attachment_id) + if item is None or str(item["chat_id"]) != chat_id: + return None + if item["status"] == "failed" or item["status"] not in ("queued", "processing", "ready"): + raise AttachmentRejected() + row = conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, body, created_at) + VALUES (%s, %s, %s, %s, clock_timestamp()) + RETURNING id, chat_id, sender_id, body AS text, created_at + """, + (message_id, chat_id, sender_id, text), + ).fetchone() + for ordinal, attachment_id in enumerate(attachment_ids): + conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, ordinal) + VALUES (%s, %s, %s) + """, + (row["id"], attachment_id, ordinal), + ) + conn.execute( + """ + INSERT INTO idempotency_keys ( + user_id, http_method, request_path, idem_key, + request_hash, message_id, created_at + ) + VALUES (%s, 'POST', %s, %s, %s, %s, clock_timestamp()) + """, + (sender_id, path, idem_key, digest, row["id"]), + ) + return public_message(row, attachment_ids) + except CheckViolation as exc: + raise ValueError("message") from exc + except psycopg.OperationalError as exc: + _db_down(exc) + + def list_messages(self, chat_id, user_id, limit, cursor): + after = None + if cursor is not None: + created_at, item_id = decode_cursor(cursor, chat_id) + after = (parse_time(created_at), uuid.UUID(item_id)) + try: + with self._connect() as conn: + if self._member(conn, chat_id, user_id) is None: + return None + if after is None: + rows = conn.execute(SQL_MESSAGES_FIRST, (chat_id, limit + 1)).fetchall() + else: + rows = conn.execute( + SQL_MESSAGES_AFTER, + (chat_id, after[0], after[1], limit + 1), + ).fetchall() + linked = self._attachment_ids(conn, [row["id"] for row in rows[:limit]]) + except psycopg.OperationalError as exc: + _db_down(exc) + page = [public_message(row, linked.get(str(row["id"]), [])) for row in rows[:limit]] + if len(rows) > limit and page: + last = page[-1] + return page, encode_cursor(chat_id, last["created_at"], last["id"]) + return page, None diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..fec68d8 --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,448 @@ +"""OpenTelemetry: traces, metrics, structured logs. Секреты и тела сообщений не экспортируются.""" +from __future__ import annotations + +import json +import logging +import os +import socket +import time +from contextlib import contextmanager + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.metrics import CallbackOptions, Observation +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ALWAYS_ON, ParentBased, TraceIdRatioBased +from opentelemetry.trace import Link, SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +SERVICE_VERSION = "lab6" +PROPAGATOR = TraceContextTextMapPropagator() +_CONFIGURED = False +_TRACER = None +_METER = None +_STATE = { + "queue_depth": {}, + "oldest_job_age": {}, + "db_pool_used": 0.0, + "db_pool_waiting": 0.0, + "worker_busy": 0.0, + "worker_capacity": 0.0, + "process_cpu": 0.0, + "process_rss": 0.0, +} + +http_requests = None +http_duration = None +jobs_processed = None +job_duration = None +e2e_ready = None + + +def _endpoint(): + return os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") + + +def _headers(): + raw = os.environ.get("OTEL_EXPORTER_OTLP_HEADERS", "") + if not raw: + return {} + out = {} + for part in raw.split(","): + if "=" not in part: + continue + key, value = part.split("=", 1) + out[key.strip()] = value.strip() + return out + + +def _sampler(): + ratio = os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0") + try: + value = float(ratio) + except ValueError: + value = 1.0 + if value >= 1.0: + return ParentBased(ALWAYS_ON) + if value <= 0.0: + return ParentBased(TraceIdRatioBased(0.0)) + return ParentBased(TraceIdRatioBased(value)) + + +def _resource(service_name): + attrs = { + "service.name": service_name, + "service.version": os.environ.get("SERVICE_VERSION", SERVICE_VERSION), + "service.instance.id": socket.gethostname(), + "deployment.environment.name": os.environ.get("APP_ENV", "lab"), + } + extra = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "") + for part in extra.split(","): + if "=" not in part: + continue + key, value = part.split("=", 1) + key = key.strip() + value = value.strip() + if key and value and key not in attrs: + attrs[key] = value + return Resource.create(attrs) + + +def _obs_queue_depth(_options: CallbackOptions): + for name, value in list(_STATE["queue_depth"].items()): + yield Observation(float(value), {"messaging.destination.name": name}) + + +def _obs_oldest(_options: CallbackOptions): + for name, value in list(_STATE["oldest_job_age"].items()): + yield Observation(float(value), {"messaging.destination.name": name}) + + +def _obs_db_used(_options: CallbackOptions): + yield Observation(float(_STATE["db_pool_used"])) + + +def _obs_db_waiting(_options: CallbackOptions): + yield Observation(float(_STATE["db_pool_waiting"])) + + +def _obs_worker_busy(_options: CallbackOptions): + yield Observation(float(_STATE["worker_busy"])) + + +def _obs_worker_capacity(_options: CallbackOptions): + yield Observation(float(_STATE["worker_capacity"])) + + +def _obs_cpu(_options: CallbackOptions): + yield Observation(float(_STATE["process_cpu"])) + + +def _obs_rss(_options: CallbackOptions): + yield Observation(float(_STATE["process_rss"])) + + +def attach_log_handler(logger_provider): + handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + root = logging.getLogger() + root.setLevel(logging.INFO) + if not any(isinstance(item, LoggingHandler) for item in root.handlers): + root.addHandler(handler) + for name in ("messenger", "worker", "uvicorn", "uvicorn.error", "uvicorn.access"): + logging.getLogger(name).setLevel(logging.INFO) + + +def setup(service_name): + global _CONFIGURED, _TRACER, _METER + global http_requests, http_duration, jobs_processed, job_duration, e2e_ready + if _CONFIGURED: + # uvicorn мог переустановить handlers после импорта модуля + provider = getattr(setup, "_logger_provider", None) + if provider is not None: + attach_log_handler(provider) + return + if os.environ.get("OTEL_SDK_DISABLED", "").lower() in ("1", "true", "yes"): + _CONFIGURED = True + return + + resource = _resource(service_name) + base = _endpoint() + hdr = _headers() + + provider = TracerProvider(resource=resource, sampler=_sampler()) + provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter(endpoint=f"{base}/v1/traces", headers=hdr, timeout=5), + max_queue_size=512, + max_export_batch_size=128, + schedule_delay_millis=2000, + export_timeout_millis=5000, + ) + ) + trace.set_tracer_provider(provider) + _TRACER = trace.get_tracer("messenger", SERVICE_VERSION) + + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{base}/v1/metrics", headers=hdr, timeout=5), + export_interval_millis=10000, + export_timeout_millis=5000, + ) + views = ( + View( + instrument_name="messenger.http.duration", + aggregation=ExplicitBucketHistogramAggregation( + [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0] + ), + ), + View( + instrument_name="messenger.jobs.duration", + aggregation=ExplicitBucketHistogramAggregation( + [0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 15.0, 30.0, 60.0] + ), + ), + View( + instrument_name="messenger.attachment.time_to_ready", + aggregation=ExplicitBucketHistogramAggregation( + [0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0] + ), + ), + ) + meter_provider = MeterProvider(resource=resource, metric_readers=[reader], views=views) + metrics.set_meter_provider(meter_provider) + _METER = metrics.get_meter("messenger", SERVICE_VERSION) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=f"{base}/v1/logs", headers=hdr, timeout=5), + max_queue_size=512, + max_export_batch_size=128, + schedule_delay_millis=2000, + export_timeout_millis=5000, + ) + ) + set_logger_provider(logger_provider) + setup._logger_provider = logger_provider + attach_log_handler(logger_provider) + + http_requests = _METER.create_counter( + "messenger.http.requests", + unit="1", + description="HTTP requests by method, route template and status class", + ) + http_duration = _METER.create_histogram( + "messenger.http.duration", + unit="s", + description="HTTP request duration in seconds", + ) + jobs_processed = _METER.create_counter( + "messenger.jobs.processed", + unit="1", + description="Jobs finished by type and outcome", + ) + job_duration = _METER.create_histogram( + "messenger.jobs.duration", + unit="s", + description="Job processing duration in seconds", + ) + e2e_ready = _METER.create_histogram( + "messenger.attachment.time_to_ready", + unit="s", + description="Seconds from durable acceptance to ready", + ) + _METER.create_observable_up_down_counter( + "messenger.queue.depth", + callbacks=[_obs_queue_depth], + unit="1", + description="Approximate broker queue depth", + ) + _METER.create_observable_gauge( + "messenger.queue.oldest_job_age", + callbacks=[_obs_oldest], + unit="s", + description="Age of the oldest unfinished job", + ) + _METER.create_observable_gauge( + "messenger.db.pool.used", + callbacks=[_obs_db_used], + unit="1", + description="Checked-out DB pool connections", + ) + _METER.create_observable_gauge( + "messenger.db.pool.waiting", + callbacks=[_obs_db_waiting], + unit="1", + description="Waiters for a DB pool connection", + ) + _METER.create_observable_gauge( + "messenger.worker.busy", + callbacks=[_obs_worker_busy], + unit="1", + description="Busy worker slots", + ) + _METER.create_observable_gauge( + "messenger.worker.capacity", + callbacks=[_obs_worker_capacity], + unit="1", + description="Configured worker concurrency", + ) + _METER.create_observable_gauge( + "messenger.process.cpu_seconds", + callbacks=[_obs_cpu], + unit="s", + description="Process user+system CPU seconds", + ) + _METER.create_observable_gauge( + "messenger.process.rss_bytes", + callbacks=[_obs_rss], + unit="By", + description="Process RSS bytes", + ) + _CONFIGURED = True + + +def tracer(): + return _TRACER or trace.get_tracer("messenger", SERVICE_VERSION) + + +def status_class(code): + return f"{int(code) // 100}xx" + + +def route_template(path): + if path in ("/health/live", "/health/ready"): + return path + parts = [p for p in path.strip("/").split("/") if p] + if len(parts) >= 3 and parts[0] == "api" and parts[1] == "v1": + if parts[2] == "users" and len(parts) == 3: + return "/api/v1/users" + if parts[2] == "users" and len(parts) == 4 and parts[3] == "me": + return "/api/v1/users/me" + if parts[2] == "auth" and len(parts) >= 4 and parts[3] == "sessions": + if len(parts) == 4: + return "/api/v1/auth/sessions" + return "/api/v1/auth/sessions/current" + if parts[2] == "chats" and len(parts) == 3: + return "/api/v1/chats" + if parts[2] == "chats" and len(parts) == 4: + return "/api/v1/chats/{chat_id}" + if parts[2] == "chats" and len(parts) == 5 and parts[4] == "messages": + return "/api/v1/chats/{chat_id}/messages" + if parts[2] == "chats" and len(parts) == 5 and parts[4] == "attachments": + return "/api/v1/chats/{chat_id}/attachments" + if parts[2] == "attachments" and len(parts) == 4: + return "/api/v1/attachments/{attachment_id}" + if parts[2] == "attachments" and len(parts) == 5 and parts[4] == "content": + return "/api/v1/attachments/{attachment_id}/content" + return "other" + + +def record_http(method, path, status, duration_s): + if http_requests is None: + return + if path in ("/health/live", "/health/ready"): + return + route = route_template(path) + attrs = { + "http.request.method": method, + "http.route": route, + "http.response.status_class": status_class(status), + } + http_requests.add(1, attrs) + http_duration.record(duration_s, attrs) + + +def inject_trace(carrier): + PROPAGATOR.inject(carrier) + return carrier + + +def extract_context(carrier): + return PROPAGATOR.extract(carrier) + + +def current_trace_fields(): + span = trace.get_current_span() + ctx = span.get_span_context() + if not ctx.is_valid: + return {}, None, None + carrier = {} + inject_trace(carrier) + return carrier, format(ctx.trace_id, "032x"), format(ctx.span_id, "016x") + + +def structured_log(logger, event, level=logging.INFO, **fields): + _carrier, trace_id, span_id = current_trace_fields() + payload = { + "event": event, + "service.name": os.environ.get("OTEL_SERVICE_NAME", "messenger"), + } + if trace_id: + payload["trace_id"] = trace_id + if span_id: + payload["span_id"] = span_id + for key, value in fields.items(): + if value is None: + continue + if key in ("password", "token", "authorization", "body", "text", "bytes"): + continue + payload[key] = value + logger.log(level, json.dumps(payload, ensure_ascii=False, sort_keys=True)) + + +@contextmanager +def start_span(name, kind=SpanKind.INTERNAL, attributes=None, links=None, context=None): + attrs = dict(attributes or {}) + span = tracer().start_span(name, context=context, kind=kind, attributes=attrs, links=links or []) + token = trace.use_span(span, end_on_exit=True) + token.__enter__() + try: + yield span + except Exception as exc: + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + raise + finally: + token.__exit__(None, None, None) + + +def link_from_carrier(carrier): + ctx = extract_context(carrier) + span_ctx = trace.get_current_span(ctx).get_span_context() + if span_ctx.is_valid: + return [Link(span_ctx)] + return [] + + +def observe_process(): + try: + import resource + + usage = resource.getrusage(resource.RUSAGE_SELF) + _STATE["process_rss"] = float(usage.ru_maxrss) * 1024.0 + _STATE["process_cpu"] = float(usage.ru_utime + usage.ru_stime) + except Exception: + pass + + +def set_pool_stats(used, waiting): + _STATE["db_pool_used"] = float(used) + _STATE["db_pool_waiting"] = float(waiting) + + +def set_worker_stats(busy, capacity): + _STATE["worker_busy"] = float(busy) + _STATE["worker_capacity"] = float(capacity) + + +def set_queue_stats(name, depth, oldest_age_s): + _STATE["queue_depth"][name] = int(depth) + _STATE["oldest_job_age"][name] = float(oldest_age_s) + + +def record_job(outcome, duration_s, job_type="attachment.process.v1"): + if jobs_processed is None: + return + attrs = {"job.type": job_type, "job.outcome": outcome} + jobs_processed.add(1, attrs) + job_duration.record(duration_s, attrs) + + +def record_time_to_ready(seconds): + if e2e_ready is None: + return + e2e_ready.record(float(seconds), {"job.type": "attachment.process.v1"}) + + +def now_monotonic(): + return time.perf_counter() diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..4ce7949 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,241 @@ +"""Отдельный процесс. API не уменьшает картинку, даже если worker остановлен.""" +import logging +import os +import socket +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone + +import pika +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from blobs import bucket_ready, read_secret +from broker import PROCESS_QUEUE, RETRY_QUEUE, _parameters, broker_channel, ensure_topology, publish_pending, queue_depth +from jobs import cleanup_staging, lease_seconds, process_delivery, release_expired +from telemetry import observe_process, set_queue_stats, set_worker_stats, setup as setup_telemetry, structured_log + +LOGGER = logging.getLogger("worker") +SCHEMA_VERSION = "004_jobs" +BUSY = 0 +BUSY_LOCK = threading.Lock() + + +def database_url(): + return read_secret(os.environ["DATABASE_URL_FILE"]) + + +def concurrency(): + value = int(os.environ.get("WORKER_CONCURRENCY", "2")) + if not 1 <= value <= 8: + raise RuntimeError("WORKER_CONCURRENCY") + return value + + +def wait_schema(conninfo): + last = "error" + for _ in range(90): + try: + with psycopg.connect(conninfo, connect_timeout=3, row_factory=dict_row) as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (SCHEMA_VERSION,), + ).fetchone() + if row is not None: + return + last = "migration missing" + except psycopg.Error as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("schema not ready: %s", last) + raise SystemExit(1) + + +def wait_bucket(): + for _ in range(60): + if bucket_ready(): + return + time.sleep(1) + LOGGER.error("object storage not ready") + raise SystemExit(1) + + +def wait_broker(): + last = "error" + for _ in range(60): + try: + with broker_channel(): + return + except Exception as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("broker not ready: %s", last) + raise SystemExit(1) + + +def make_pool(conninfo, slots): + return ConnectionPool( + conninfo, + min_size=1, + max_size=slots * 2 + 2, + timeout=3, + kwargs={ + "connect_timeout": 3, + "row_factory": dict_row, + "application_name": "mtusi-lab6-worker", + "options": "-c statement_timeout=5000 -c lock_timeout=3000 -c idle_in_transaction_session_timeout=30000", + }, + open=True, + ) + + +def oldest_job_age_seconds(pool): + try: + with pool.connection() as conn: + row = conn.execute( + """ + SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) AS age + FROM attachment_jobs + WHERE status IN ('queued', 'processing') + """ + ).fetchone() + if row is None or row["age"] is None: + return 0.0 + return float(row["age"]) + except Exception: + return 0.0 + + +def refresh_runtime_stats(pool, slots): + global BUSY + try: + depth = queue_depth(PROCESS_QUEUE) + except Exception: + depth = 0 + try: + retry_depth = queue_depth(RETRY_QUEUE) + except Exception: + retry_depth = 0 + age = oldest_job_age_seconds(pool) + set_queue_stats(PROCESS_QUEUE, depth, age) + set_queue_stats(RETRY_QUEUE, retry_depth, 0.0) + with BUSY_LOCK: + busy = BUSY + set_worker_stats(busy, slots) + observe_process() + try: + stats = pool.get_stats() + from telemetry import set_pool_stats + + used = int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0)) + waiting = int(stats.get("requests_waiting", 0)) + set_pool_stats(max(0, used), waiting) + except Exception: + pass + + +def consume(pool, owner, slots): + executor = ThreadPoolExecutor(max_workers=slots, thread_name_prefix="job") + while True: + connection = None + try: + connection = pika.BlockingConnection(_parameters()) + channel = connection.channel() + ensure_topology(channel) + channel.basic_qos(prefetch_count=slots) + + def on_message(ch, method, _props, body): + executor.submit(run_one, pool, owner, ch, method.delivery_tag, body) + + channel.basic_consume(PROCESS_QUEUE, on_message, auto_ack=False) + channel.start_consuming() + except Exception as exc: + LOGGER.warning("consumer %s", type(exc).__name__) + time.sleep(1) + finally: + if connection is not None: + try: + connection.close() + except Exception: + pass + + +def run_one(pool, owner, channel, tag, body): + global BUSY + with BUSY_LOCK: + BUSY += 1 + try: + process_delivery(pool, body, owner) + action = "ack" + except Exception as exc: + LOGGER.warning("delivery %s", type(exc).__name__) + action = "nack" + finally: + with BUSY_LOCK: + BUSY = max(0, BUSY - 1) + + def finish(): + try: + if action == "ack": + channel.basic_ack(tag) + else: + channel.basic_nack(tag, requeue=True) + except Exception: + pass + + try: + channel.connection.add_callback_threadsafe(finish) + except Exception: + pass + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker")) + conninfo = database_url() + wait_schema(conninfo) + wait_bucket() + wait_broker() + slots = concurrency() + pool = make_pool(conninfo, slots) + owner = socket.gethostname() + stop = threading.Event() + set_worker_stats(0, slots) + + def drain(): + while not stop.wait(1): + try: + with pool.connection() as conn: + publish_pending(conn) + except Exception as exc: + LOGGER.warning("drain %s", type(exc).__name__) + + def sweep(): + ticks = 0 + while not stop.wait(5): + ticks += 1 + try: + with pool.connection() as conn: + release_expired(conn) + if ticks % 12 == 0: + cleanup_staging(conn) + refresh_runtime_stats(pool, slots) + except Exception as exc: + LOGGER.warning("sweep %s", type(exc).__name__) + + threading.Thread(target=drain, name="drain", daemon=True).start() + threading.Thread(target=sweep, name="sweep", daemon=True).start() + try: + with pool.connection() as conn: + publish_pending(conn) + except Exception as exc: + LOGGER.warning("startup drain %s", type(exc).__name__) + structured_log(LOGGER, "worker_ready", owner=owner, concurrency=slots, lease=lease_seconds()) + LOGGER.info("worker ready owner=%s concurrency=%s lease=%s", owner, slots, lease_seconds()) + consume(pool, owner, slots) + + +if __name__ == "__main__": + main() diff --git a/student-tests/probe_loki.py b/student-tests/probe_loki.py new file mode 100644 index 0000000..2586f2f --- /dev/null +++ b/student-tests/probe_loki.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +import json +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=str(ROOT), + capture_output=True, + text=True, + ) + + +def main(): + try: + urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read() + except urllib.error.HTTPError: + pass + time.sleep(10) + end = int(time.time()) + start = end - 1800 + for query in ( + '{service_name="messenger-api"}', + '{service_name="messenger-worker"}', + '{service_name=~".+"}', + ): + url = "http://127.0.0.1:3100/loki/api/v1/query_range?" + urllib.parse.urlencode( + { + "query": query, + "start": str(start) + "000000000", + "end": str(end) + "000000000", + "limit": "20", + } + ) + out = compose("exec", "-T", "loki", "wget", "-qO-", url) + try: + data = json.loads(out.stdout) + streams = data.get("data", {}).get("result", []) + lines = sum(len(item.get("values", [])) for item in streams) + print(query, "streams", len(streams), "lines", lines) + if streams and streams[0].get("values"): + print("sample", streams[0]["values"][0][1][:240]) + except Exception as exc: + print(query, "err", exc, out.stdout[:200], out.stderr[:200]) + print("collector tail:") + print(compose("logs", "otel-collector", "--since", "2m").stdout[-2000:]) + q = '{service_name="messenger-api"} |= "http_request"' + url = "http://127.0.0.1:3100/loki/api/v1/query_range?" + urllib.parse.urlencode( + { + "query": q, + "start": str(start) + "000000000", + "end": str(end) + "000000000", + "limit": "10", + } + ) + out = compose("exec", "-T", "loki", "wget", "-qO-", url) + data = json.loads(out.stdout) + streams = data.get("data", {}).get("result", []) + print("http_request streams", len(streams), "lines", sum(len(s.get("values", [])) for s in streams)) + for stream in streams[:3]: + for value in stream.get("values", [])[:2]: + print(value[1][:400]) + + +if __name__ == "__main__": + main() diff --git a/student-tests/probe_traces.py b/student-tests/probe_traces.py new file mode 100644 index 0000000..083d150 --- /dev/null +++ b/student-tests/probe_traces.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +import base64 +import json +import subprocess +import time +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = "http://127.0.0.1:8080" + + +def compose(*args): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=str(ROOT), + capture_output=True, + text=True, + ) + + +def http(method, path, body=None, headers=None): + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request(BASE + path, data=data, method=method, headers=headers or {}) + if data is not None: + req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read() + payload = json.loads(raw.decode()) if raw else None + return resp.status, dict(resp.headers), payload + + +def main(): + stamp = uuid.uuid4().hex[:10] + users = [] + for idx in range(2): + username = f"tr_{stamp}_{idx}" + http("POST", "/api/v1/users", {"username": username, "display_name": username, "password": "Password123!"}) + auth = "Basic " + base64.b64encode(f"{username}:Password123!".encode()).decode() + _status, _headers, session = http("POST", "/api/v1/auth/sessions", headers={"Authorization": auth}) + users.append(session) + _status, _headers, chat = http( + "POST", + "/api/v1/chats", + {"title": "t", "member_ids": [users[0]["user"]["id"], users[1]["user"]["id"]]}, + headers={"Authorization": "Bearer " + users[0]["token"]}, + ) + png = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + req = urllib.request.Request( + BASE + f"/api/v1/chats/{chat['id']}/attachments?filename=t.png", + data=png, + method="POST", + headers={"Authorization": "Bearer " + users[0]["token"], "Content-Type": "image/png"}, + ) + with urllib.request.urlopen(req) as resp: + attachment = json.loads(resp.read().decode()) + request_id = resp.headers.get("X-Request-Id") + print("upload", resp.status, attachment["id"], request_id) + for _ in range(60): + _status, _headers, payload = http( + "GET", + f"/api/v1/attachments/{attachment['id']}", + headers={"Authorization": "Bearer " + users[0]["token"]}, + ) + if payload.get("status") == "ready": + print("ready") + break + time.sleep(0.5) + time.sleep(6) + for service in ("messenger-api", "messenger-worker"): + q = urllib.parse.urlencode({"tags": f"service.name={service}", "limit": "20"}) + out = compose("exec", "-T", "tempo", "wget", "-qO-", f"http://127.0.0.1:3200/api/search?{q}") + print(service, out.stdout[:800]) + q = urllib.parse.urlencode({"query": '{service_name="messenger-api"} |= "http_request"', "limit": "5"}) + out = compose("exec", "-T", "loki", "wget", "-qO-", "http://127.0.0.1:3100/loki/api/v1/query?" + q) + print("loki", out.stdout[:600]) + # worker stop age probe + compose("stop", "worker") + req = urllib.request.Request( + BASE + f"/api/v1/chats/{chat['id']}/attachments?filename=lag.png", + data=png, + method="POST", + headers={"Authorization": "Bearer " + users[0]["token"], "Content-Type": "image/png"}, + ) + with urllib.request.urlopen(req) as resp: + queued = json.loads(resp.read().decode()) + print("queued", resp.status, queued["id"]) + ages = [] + fired = None + started = time.time() + while time.time() - started < 150: + q = urllib.parse.urlencode( + { + "query": 'max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"})' + } + ) + out = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/query?" + q) + age = None + try: + payload = json.loads(out.stdout) + rows = payload.get("data", {}).get("result", []) + if rows: + age = float(rows[0]["value"][1]) + except Exception: + age = None + ages.append({"t": round(time.time() - started, 1), "age": age}) + alerts = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/alerts") + try: + alert_payload = json.loads(alerts.stdout) + for item in alert_payload.get("data", {}).get("alerts", []): + if item.get("labels", {}).get("alertname") == "GrowingQueueLag" and item.get("state") == "firing": + fired = round(time.time() - started, 1) + break + except Exception: + pass + if fired is not None: + break + time.sleep(5) + compose("start", "worker") + print(json.dumps({"ages": ages[-8:], "fired_after_seconds": fired}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_negative.py b/student-tests/run_negative.py new file mode 100644 index 0000000..da74c89 --- /dev/null +++ b/student-tests/run_negative.py @@ -0,0 +1,136 @@ +"""Негативные проверки API. Токены и пароли не печатаются.""" +import json +import struct +import sys +import uuid +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def png(width, height, payload=b""): + def chunk(tag, data): + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + payload + chunk(b"IEND", b"") + + +def main(): + base = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8080" + client = Client(base) + alice = client.user("n1") + bob = client.user("n2") + eve = client.user("n3") + chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Негативные", "member_ids": [alice["id"].upper(), bob["id"]]}, + schema="Chat", + )["json"] + other = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Другой", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + upper = client.call("GET", "/api/v1/chats/" + chat["id"].upper(), actor=bob, schema="Chat")["json"] + results = [] + + def record(name, status): + results.append({"name": name, "status": status}) + + record("chat_uuid_case", 200 if upper["id"] == chat["id"] else 0) + path = "/api/v1/chats/" + chat["id"] + "/messages" + record("blank_text", client.call("POST", path, 400, actor=alice, body={"text": " "}, headers={"Idempotency-Key": "blank-1"})["status"]) + record("bad_json", client.call("POST", path, 400, actor=alice, raw=b"{", headers={"Content-Type": "application/json", "Idempotency-Key": "bad-json"})["status"]) + record("short_password", client.call("POST", "/api/v1/users", 400, body={"username": "abc", "display_name": "Имя", "password": "short"})["status"]) + key = uuid.uuid4().hex + first = client.call("POST", path, 201, actor=alice, body={"text": "одно"}, headers={"Idempotency-Key": key}, schema="Message")["json"] + second = client.call( + "POST", + path, + 201, + actor=alice, + body={"text": "одно", "attachment_ids": []}, + headers={"Idempotency-Key": key}, + schema="Message", + )["json"] + record("omitted_equals_empty", 201 if first == second else 0) + record("same_key_other_body", client.call("POST", path, 409, actor=alice, body={"text": "другое"}, headers={"Idempotency-Key": key})["status"]) + history = client.call("GET", path.replace("/messages", "/messages?limit=100"), actor=bob, schema="MessagePage")["json"] + record("no_second_row", 200 if sum(1 for item in history["items"] if item["text"] in ("одно", "другое")) == 1 else 0) + other_path = "/api/v1/chats/" + other["id"] + "/messages" + other_message = client.call( + "POST", + other_path, + 201, + actor=alice, + body={"text": "одно"}, + headers={"Idempotency-Key": key}, + schema="Message", + )["json"] + record("same_key_other_chat", 201 if other_message["id"] != first["id"] else 0) + record("foreign_chat", client.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)["status"]) + record("bad_cursor", client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?cursor=not-a-cursor", 400, actor=alice)["status"]) + upload = "/api/v1/chats/" + chat["id"] + "/attachments" + record("slash_name", client.call("POST", upload + "?filename=a/b.png", 400, actor=alice, raw=png(1, 1), headers={"Content-Type": "image/png"})["status"]) + record("huge_pixels", client.call("POST", upload + "?filename=big.png", 413, actor=alice, raw=png(8000, 8000), headers={"Content-Type": "image/png"})["status"]) + record("bad_png", client.call("POST", upload + "?filename=bad.png", 415, actor=alice, raw=b"\x89PNG\r\n\x1a\nbroken", headers={"Content-Type": "image/png"})["status"]) + record("type_mismatch", client.call("POST", upload + "?filename=no.jpg", 415, actor=alice, raw=png(1, 1), headers={"Content-Type": "image/jpeg"})["status"]) + record("oversize", client.call("POST", upload + "?filename=over.png", 413, actor=alice, raw=b"x" * (10 * 1024 * 1024 + 1), headers={"Content-Type": "image/png"})["status"]) + missing = str(uuid.uuid4()) + record("unknown_attachment", client.call("GET", "/api/v1/attachments/" + missing.upper(), 404, actor=alice)["status"]) + record("no_token", client.call("GET", "/api/v1/users/me", 401)["status"]) + payload = { + "command": "python -X utf8 student-tests/run_negative.py", + "result": "PASS negative", + "results": results, + } + body = json.dumps(payload, ensure_ascii=False, indent=2) + print(body) + failed = [item for item in results if item["status"] not in (200, 201, 400, 401, 404, 409, 413, 415)] + if failed: + return 1 + expected = { + "chat_uuid_case": 200, + "blank_text": 400, + "bad_json": 400, + "short_password": 400, + "omitted_equals_empty": 201, + "same_key_other_body": 409, + "no_second_row": 200, + "same_key_other_chat": 201, + "foreign_chat": 404, + "bad_cursor": 400, + "slash_name": 400, + "huge_pixels": 413, + "bad_png": 415, + "type_mismatch": 415, + "oversize": 413, + "unknown_attachment": 404, + "no_token": 401, + } + for name, status in expected.items(): + got = next(item["status"] for item in results if item["name"] == name) + if got != status: + print("mismatch " + name, file=sys.stderr) + return 1 + (ROOT / "evidence" / "negative.json").write_text(body + "\n", encoding="utf-8") + print("PASS negative") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Failure as exc: + print("FAIL " + str(exc), file=sys.stderr) + raise SystemExit(1) diff --git a/student-tests/run_observability.py b/student-tests/run_observability.py new file mode 100644 index 0000000..b609587 --- /dev/null +++ b/student-tests/run_observability.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 +"""Учебные проверки наблюдаемости и инцидентов. Пароли не печатает.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +OUT = ROOT / "evidence" / "observability.json" + + +def http_json(method, path, body=None, headers=None, timeout=30): + data = None if body is None else json.dumps(body).encode("utf-8") + req = urllib.request.Request( + BASE + path, + data=data, + method=method, + headers=headers or {}, + ) + if data is not None: + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read() + payload = json.loads(raw.decode("utf-8")) if raw else None + return resp.status, dict(resp.headers), payload + except urllib.error.HTTPError as exc: + raw = exc.read() + payload = None + if raw: + try: + payload = json.loads(raw.decode("utf-8")) + except ValueError: + payload = {"raw": raw.decode("utf-8", errors="replace")[:200]} + return exc.code, dict(exc.headers), payload + + +def compose(*args): + cmd = ["docker", "compose", "--env-file", ".env", *args] + return subprocess.run(cmd, cwd=str(ROOT), capture_output=True, text=True, check=False) + + +def prom_query(expr): + q = urllib.parse.urlencode({"query": expr}) + url = "http://127.0.0.1:9090/api/v1/query?" + q + # Prometheus не опубликован наружу. Читаем через docker exec. + result = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/query?" + q) + if result.returncode != 0: + return None + try: + payload = json.loads(result.stdout) + except ValueError: + return None + rows = payload.get("data", {}).get("result", []) + if not rows: + return 0.0 + return float(rows[0]["value"][1]) + + +def prom_alerts(): + result = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/alerts") + if result.returncode != 0: + return [] + try: + payload = json.loads(result.stdout) + except ValueError: + return [] + return payload.get("data", {}).get("alerts", []) + + +def tempo_search(service, name=None, limit=10): + # Tag search + TraceQL fallback. + tags = "service.name=" + service + q = urllib.parse.urlencode({"tags": tags, "limit": str(limit)}) + result = compose("exec", "-T", "tempo", "wget", "-qO-", "http://127.0.0.1:3200/api/search?" + q) + traces = [] + if result.returncode == 0: + try: + traces = json.loads(result.stdout).get("traces", []) + except ValueError: + traces = [] + if name: + traces = [t for t in traces if name in (t.get("rootTraceName") or "")] + if traces: + return traces + query = f'{{resource.service.name="{service}"}}' + if name: + query = f'{{resource.service.name="{service}" && name="{name}"}}' + q = urllib.parse.urlencode({"q": query, "limit": str(limit)}) + result = compose("exec", "-T", "tempo", "wget", "-qO-", "http://127.0.0.1:3200/api/search?" + q) + if result.returncode != 0: + return [] + try: + return json.loads(result.stdout).get("traces", []) + except ValueError: + return [] + + +def register_and_chat(): + stamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S%f") + users = [] + for idx in range(2): + username = f"obs_{stamp}_{idx}" + status, _, user = http_json( + "POST", + "/api/v1/users", + {"username": username, "display_name": username, "password": "Password123!"}, + ) + assert status == 201, status + status, _, session = http_json( + "POST", + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + __import__("base64").b64encode(f"{username}:Password123!".encode()).decode()}, + ) + assert status == 201, status + users.append({"user": user, "token": session["token"]}) + status, _, chat = http_json( + "POST", + "/api/v1/chats", + {"title": "obs", "member_ids": [users[0]["user"]["id"], users[1]["user"]["id"]]}, + headers={"Authorization": "Bearer " + users[0]["token"]}, + ) + assert status == 201, status + return users, chat + + +def upload_png(token, chat_id, name="probe.png"): + data = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + req = urllib.request.Request( + f"{BASE}/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(name)}", + data=data, + method="POST", + headers={ + "Authorization": "Bearer " + token, + "Content-Type": "image/png", + "Content-Length": str(len(data)), + }, + ) + with urllib.request.urlopen(req, timeout=30) as resp: + payload = json.loads(resp.read().decode("utf-8")) + return resp.status, dict(resp.headers), payload + + +def api_direct(method, path, token=None, timeout=3): + """Запрос внутрь контейнера API, минуя HAProxy. Нужен, когда ready уже красный.""" + headers = ["-H", "Accept: application/json"] + if token: + headers.extend(["-H", f"Authorization: Bearer {token}"]) + cmd = [ + "exec", + "-T", + "api", + "python", + "-c", + ( + "import os,urllib.request,sys\n" + f"req=urllib.request.Request('http://127.0.0.1:8080{path}', method='{method}')\n" + ), + ] + # Проще через wget/python one-liner in api container. + py = ( + "import urllib.request,urllib.error\n" + f"req=urllib.request.Request('http://127.0.0.1:8080{path}', method='{method}')\n" + ) + if token: + py += f"req.add_header('Authorization', 'Bearer {token}')\n" + py += ( + "try:\n" + f" r=urllib.request.urlopen(req, timeout={timeout})\n" + " print(r.status)\n" + "except urllib.error.HTTPError as e:\n" + " print(e.code)\n" + "except Exception as e:\n" + " print(type(e).__name__)\n" + ) + result = compose("exec", "-T", "api", "python", "-c", py) + text = (result.stdout or "").strip().splitlines() + return text[-1] if text else "empty" + + +def wait_ready_attachment(token, attachment_id, timeout=60): + deadline = time.time() + timeout + while time.time() < deadline: + status, _, payload = http_json( + "GET", + f"/api/v1/attachments/{attachment_id}", + headers={"Authorization": "Bearer " + token}, + ) + if status == 200 and payload.get("status") == "ready": + return payload + time.sleep(0.5) + raise TimeoutError(attachment_id) + + +def scenario_trace_link(): + users, chat = register_and_chat() + token = users[0]["token"] + status, headers, attachment = upload_png(token, chat["id"]) + assert status == 202, status + ready = wait_ready_attachment(token, attachment["id"]) + time.sleep(3) + api_traces = tempo_search("messenger-api", "POST /api/v1/chats/{chat_id}/attachments") + worker_traces = tempo_search("messenger-worker", "attachments.process consume") + return { + "ok": True, + "attachment_id": attachment["id"], + "attachment_status": ready["status"], + "request_id": headers.get("X-Request-Id") or headers.get("x-request-id"), + "api_upload_traces": len(api_traces), + "worker_consume_traces": len(worker_traces), + "sample_api_trace": api_traces[0]["traceID"] if api_traces else None, + "sample_worker_trace": worker_traces[0]["traceID"] if worker_traces else None, + } + + +def scenario_worker_stop(): + users, chat = register_and_chat() + token = users[0]["token"] + compose("stop", "worker") + time.sleep(2) + status, _, attachment = upload_png(token, chat["id"], "queued.png") + assert status == 202, status + started = time.time() + fired_at = None + age_samples = [] + while time.time() - started < 90: + age = prom_query('max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"})') + age_samples.append({"t": round(time.time() - started, 1), "age": age}) + alerts = [a for a in prom_alerts() if a.get("labels", {}).get("alertname") == "GrowingQueueLag" and a.get("state") == "firing"] + if alerts and fired_at is None: + fired_at = round(time.time() - started, 1) + break + time.sleep(5) + compose("start", "worker") + ready = wait_ready_attachment(token, attachment["id"], timeout=120) + resolved_at = None + recover_started = time.time() + while time.time() - recover_started < 120: + alerts = [a for a in prom_alerts() if a.get("labels", {}).get("alertname") == "GrowingQueueLag" and a.get("state") == "firing"] + if not alerts and fired_at is not None: + resolved_at = round(time.time() - recover_started, 1) + break + time.sleep(5) + return { + "ok": ready["status"] == "ready", + "attachment_id": attachment["id"], + "fired_after_seconds": fired_at, + "resolved_after_seconds": resolved_at, + "age_samples": age_samples[-6:], + "final_age": prom_query('messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"}'), + } + + +def scenario_db_stop(): + # Отказ Postgres. Через HAProxy ready красный, поэтому бизнес-5xx снимаю прямым + # запросом в контейнер API, чтобы метрики и traces остались от приложения. + users, chat = register_and_chat() + token = users[0]["token"] + before_5xx = prom_query('sum(messenger_http_requests_total{http_response_status_class="5xx"}) or vector(0)') + compose("stop", "postgres") + started = time.time() + business_codes = [] + ready_via_lb = [] + while time.time() - started < 90: + business_codes.append(api_direct("GET", "/api/v1/users/me", token=token)) + try: + ready_status, _, _ = http_json("GET", "/health/ready", timeout=3) + except Exception as exc: + ready_status = type(exc).__name__ + ready_via_lb.append(ready_status) + time.sleep(1) + high_fired = None + for sample_t in range(0, 90, 5): + pass + compose("start", "postgres") + deadline = time.time() + 90 + while time.time() < deadline: + try: + status, _, payload = http_json("GET", "/health/ready", timeout=3) + if status == 200 and payload and payload.get("status") == "ok": + break + except Exception: + pass + time.sleep(2) + # ждём for: 1m у HighHttpErrorRate и запись метрик + watch_started = time.time() + alerts_timeline = [] + while time.time() - watch_started < 150: + alerts = [ + a + for a in prom_alerts() + if a.get("labels", {}).get("alertname") + in ("HighHttpErrorRate", "ErrorBudgetBurnFast", "ErrorBudgetBurnSlow") + ] + alerts_timeline.append( + { + "t": round(time.time() - watch_started, 1), + "alerts": [ + {"alertname": a.get("labels", {}).get("alertname"), "state": a.get("state")} + for a in alerts + ], + } + ) + if any(a.get("state") == "firing" for a in alerts) and high_fired is None: + high_fired = round(time.time() - watch_started, 1) + if high_fired is not None and not any(a.get("state") == "firing" for a in alerts): + break + time.sleep(5) + after_5xx = prom_query('sum(messenger_http_requests_total{http_response_status_class="5xx"}) or vector(0)') + return { + "ok": True, + "business_statuses_direct_api": business_codes[:8] + ["..."] + business_codes[-4:], + "ready_via_lb": ready_via_lb[:4] + ["..."] + ready_via_lb[-3:], + "http_5xx_before": before_5xx, + "http_5xx_after": after_5xx, + "high_error_seen_after_seconds": high_fired, + "alerts_timeline_tail": alerts_timeline[-6:], + "note": "Прямые запросы в api-контейнер, потому что HAProxy при красном ready отдаёт свою 503-страницу", + } + + +def scenario_telemetry_backend_down(): + users, chat = register_and_chat() + token = users[0]["token"] + compose("stop", "tempo") + compose("stop", "loki") + compose("stop", "prometheus") + time.sleep(2) + mem_before = compose("stats", "--no-stream", "--format", "{{.MemUsage}}", "otel-collector").stdout.strip() + status, _, attachment = upload_png(token, chat["id"], "tel.png") + assert status == 202, status + # API должен жить без telemetry backends + ready_status, _, ready_body = http_json("GET", "/health/ready") + text_status, _, message = http_json( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + {"text": "telemetry backends down"}, + headers={ + "Authorization": "Bearer " + token, + "Idempotency-Key": "obs-tel-1", + }, + ) + time.sleep(8) + mem_after = compose("stats", "--no-stream", "--format", "{{.MemUsage}}", "otel-collector").stdout.strip() + compose("start", "prometheus") + compose("start", "loki") + compose("start", "tempo") + time.sleep(5) + compose("restart", "otel-collector") + wait_ready_attachment(token, attachment["id"], timeout=120) + return { + "ok": ready_status == 200 and text_status == 201, + "upload_status": status, + "ready_status": ready_status, + "message_status": text_status, + "message_id": None if not message else message.get("id"), + "otel_mem_before": mem_before, + "otel_mem_after": mem_after, + } + + +def scenario_burn_rate_spike(): + # Короткий всплеск и затем более длинная деградация для multi-window burn-rate. + users, _chat = register_and_chat() + token = users[0]["token"] + + def outage(seconds): + compose("stop", "postgres") + end = time.time() + seconds + while time.time() < end: + api_direct("GET", "/api/v1/users/me", token=token) + time.sleep(0.5) + compose("start", "postgres") + deadline = time.time() + 90 + while time.time() < deadline: + try: + status, _, payload = http_json("GET", "/health/ready", timeout=3) + if status == 200 and payload and payload.get("status") == "ok": + return + except Exception: + pass + time.sleep(2) + + # краткий всплеск ~25 с + brief_started = time.time() + outage(25) + brief_high = None + brief_burn = None + watch = time.time() + while time.time() - watch < 120: + alerts = prom_alerts() + if brief_high is None and any( + a.get("labels", {}).get("alertname") == "HighHttpErrorRate" and a.get("state") == "firing" for a in alerts + ): + brief_high = round(time.time() - brief_started, 1) + if brief_burn is None and any( + a.get("labels", {}).get("alertname") == "ErrorBudgetBurnFast" and a.get("state") == "firing" for a in alerts + ): + brief_burn = round(time.time() - brief_started, 1) + if brief_high is not None: + break + time.sleep(5) + + # длительная деградация ~6 минут + long_started = time.time() + outage(360) + long_slow = None + long_fast = None + watch = time.time() + while time.time() - watch < 240: + alerts = prom_alerts() + if long_slow is None and any( + a.get("labels", {}).get("alertname") == "ErrorBudgetBurnSlow" and a.get("state") == "firing" for a in alerts + ): + long_slow = round(time.time() - long_started, 1) + if long_fast is None and any( + a.get("labels", {}).get("alertname") == "ErrorBudgetBurnFast" and a.get("state") == "firing" for a in alerts + ): + long_fast = round(time.time() - long_started, 1) + if long_slow is not None: + break + time.sleep(5) + return { + "ok": True, + "brief_spike_seconds": 25, + "brief_high_error_fired_after_seconds": brief_high, + "brief_burn_fast_fired_after_seconds": brief_burn, + "long_degradation_seconds": 360, + "long_burn_slow_fired_after_seconds": long_slow, + "long_burn_fast_fired_after_seconds": long_fast, + "error_ratio_5m": prom_query("messenger:http_error_ratio:5m"), + "error_ratio_30m": prom_query("messenger:http_error_ratio:30m"), + "note": "Короткий всплеск может не зажечь BurnFast (нет ложного page). Длинная деградация должна зажечь BurnSlow.", + } + + +def main(): + report = { + "started_at": datetime.now(timezone.utc).isoformat(), + "base_url": BASE, + "scenarios": {}, + } + for name, func in [ + ("trace_link", scenario_trace_link), + ("worker_stop", scenario_worker_stop), + ("db_stop", scenario_db_stop), + ("telemetry_backend_down", scenario_telemetry_backend_down), + ("burn_rate_spike", scenario_burn_rate_spike), + ]: + print("RUN", name, flush=True) + try: + report["scenarios"][name] = func() + print("OK", name, flush=True) + except Exception as exc: + report["scenarios"][name] = {"ok": False, "error": type(exc).__name__, "detail": str(exc)[:300]} + print("FAIL", name, type(exc).__name__, flush=True) + report["finished_at"] = datetime.now(timezone.utc).isoformat() + OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("WROTE", OUT) + failed = [k for k, v in report["scenarios"].items() if not v.get("ok")] + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/run_pipeline.py b/student-tests/run_pipeline.py new file mode 100644 index 0000000..aab7688 --- /dev/null +++ b/student-tests/run_pipeline.py @@ -0,0 +1,653 @@ +"""Проверки очереди и трека 2. Токены и пароли не печатаются.""" +import hashlib +import json +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 +from image_probe import jpeg_size # noqa: E402 + +BASE = "http://127.0.0.1:8080" +EVIDENCE = ROOT / "evidence" / "pipeline.json" +INSIDE = r""" +import json, logging, os, hashlib +from datetime import datetime, timezone +logging.disable(logging.CRITICAL) +req = json.loads(REQUEST_JSON) +from blobs import read_secret, read_bytes, put_bytes, delete_quiet +from jobs import bump_pipeline, replay_job, republish +import psycopg +from psycopg.rows import dict_row +url = read_secret(os.environ["DATABASE_URL_FILE"]) +action = req["action"] +out = {"action": action} +with psycopg.connect(url, row_factory=dict_row) as conn: + if action == "republish": + out["ok"] = bool(republish(conn, req["job_id"], req.get("attempt"))) + elif action == "bump": + out["result"] = bump_pipeline(conn, req["attachment_id"]) + elif action == "replay": + out["result"] = replay_job(conn, req["job_id"]) + elif action == "corrupt": + row = conn.execute( + "SELECT staging_key FROM attachments WHERE id = %s", + (req["attachment_id"],), + ).fetchone() + key = row["staging_key"] + data = read_bytes(key) + stamp = datetime.now(timezone.utc) + put_bytes(key + ".held", data, "application/octet-stream", stamp) + put_bytes(key, b"not-an-image", "image/png", stamp) + out["original_sha256"] = hashlib.sha256(data).hexdigest() + out["bytes"] = len(data) + elif action == "restore": + row = conn.execute( + "SELECT staging_key FROM attachments WHERE id = %s", + (req["attachment_id"],), + ).fetchone() + key = row["staging_key"] + data = read_bytes(key + ".held") + put_bytes(key, data, "image/png", datetime.now(timezone.utc)) + delete_quiet(key + ".held") + out["restored_sha256"] = hashlib.sha256(data).hexdigest() + out["bytes"] = len(data) + else: + raise SystemExit("unknown") +print(json.dumps(out)) +""" + + +def run(args, timeout=180, check=True): + proc = subprocess.run( + args, + cwd=str(ROOT), + capture_output=True, + timeout=timeout, + ) + if check and proc.returncode != 0: + err = proc.stderr.decode("utf-8", "replace")[-1200:] + raise RuntimeError(args[0] + " failed: " + err) + return proc + + +def compose(*args, timeout=180): + run(["docker", "compose", "--env-file", ".env", *args], timeout=timeout) + + +def inspect(name, template): + proc = run(["docker", "inspect", "-f", template, name], timeout=30) + return proc.stdout.decode().strip() + + +def wait_health(name, seconds=180): + deadline = time.monotonic() + seconds + last = "" + while time.monotonic() < deadline: + last = inspect(name, "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}") + if last.startswith("running healthy"): + return last + time.sleep(1) + raise RuntimeError(name + " not healthy: " + last) + + +def restart_service(service, containers, seconds=180): + if isinstance(containers, str): + containers = [containers] + before = {name: inspect(name, "{{.State.StartedAt}}") for name in containers} + compose("restart", service) + deadline = time.monotonic() + seconds + last = "" + while time.monotonic() < deadline: + ready = True + for name in containers: + started = inspect(name, "{{.State.StartedAt}}") + last = inspect(name, "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}") + if started == before[name] or not last.startswith("running healthy"): + ready = False + if ready: + return + time.sleep(1) + raise RuntimeError(service + " did not restart: " + last) + + +def psql(sql): + proc = run( + [ + "docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-v", "ON_ERROR_STOP=1", "-q", "-t", "-A", "-c", sql, + ], + timeout=30, + ) + return proc.stdout.decode("utf-8").strip() + + +def state(attachment_id): + sql = """ + SELECT json_build_object( + 'status', a.status, + 'error_code', a.error_code, + 'pipeline_version', a.pipeline_version, + 'jobs', ( + SELECT coalesce(json_agg(json_build_object( + 'id', j.id, 'status', j.status, 'attempt', j.attempt, + 'pipeline_version', j.pipeline_version, 'error_code', j.error_code + ) ORDER BY j.pipeline_version), '[]'::json) + FROM attachment_jobs j WHERE j.attachment_id = a.id + ), + 'variants', ( + SELECT coalesce(json_agg(json_build_object( + 'name', v.name, 'pipeline_version', v.pipeline_version, + 'width', v.width, 'height', v.height, 'content_type', v.content_type + ) ORDER BY v.pipeline_version, v.name), '[]'::json) + FROM attachment_variants v WHERE v.attachment_id = a.id + ), + 'unpublished', ( + SELECT count(*) FROM outbox o + JOIN attachment_jobs j ON j.id = o.job_id + WHERE j.attachment_id = a.id AND o.published_at IS NULL + ), + 'events', ( + SELECT coalesce(json_agg(e.kind ORDER BY e.created_at), '[]'::json) + FROM job_events e WHERE e.attachment_id = a.id + ) + )::text + FROM attachments a WHERE a.id = '{id}'::uuid + """.format(id=attachment_id) + return json.loads(psql(sql)) + + +def arm(kind, attachment_id): + target = "NULL" if attachment_id is None else "'" + attachment_id + "'::uuid" + psql( + "INSERT INTO fault_injections (id, kind, attachment_id, armed) " + "VALUES (gen_random_uuid(), '{kind}', {target}, true)".format(kind=kind, target=target) + ) + + +def queues(): + proc = run( + [ + "docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", + "rabbitmqctl", "list_queues", "name", "messages", "--quiet", + ], + timeout=30, + ) + found = {} + for line in proc.stdout.decode().splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1].isdigit(): + found[parts[0]] = int(parts[1]) + return found + + +def depth(): + last = "" + for _ in range(20): + try: + found = queues() + return found.get("attachments.process", -1) + except RuntimeError as exc: + last = str(exc) + time.sleep(1) + raise RuntimeError(last) + + +def inside(req, timeout=90): + code = INSIDE.replace("REQUEST_JSON", json.dumps(json.dumps(req))) + proc = run( + [ + "docker", "compose", "--env-file", ".env", "run", "--rm", "--no-deps", "-T", + "--entrypoint", "python", "worker", "-c", code, + ], + timeout=timeout, + ) + lines = [line for line in proc.stdout.decode("utf-8").splitlines() if line.startswith("{")] + if not lines: + raise RuntimeError("no json from worker helper") + return json.loads(lines[-1]) + + +def stop_worker(): + compose("stop", "worker") + + +def start_worker(): + compose("start", "worker") + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if inspect("mtusi-lab5-worker-1", "{{.State.Status}}") == "running": + return + time.sleep(0.5) + raise RuntimeError("worker did not start") + + +def restart_count(): + return int(inspect("mtusi-lab5-worker-1", "{{.RestartCount}}")) + + +def wait_status(actor, attachment_id, want, seconds=90): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + body = client.call( + "GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment" + )["json"] + last = body["status"] + if last == want: + return body + if want != "failed" and last == "failed": + raise Failure("attachment failed " + attachment_id) + time.sleep(0.4) + raise Failure("timeout status=" + str(last) + " want=" + want) + + +def check_picture(actor, ready, original): + download = client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=actor) + if download["bytes"] != original: + raise Failure("original bytes differ") + names = {item["name"]: item for item in ready["variants"]} + thumb = client.call( + "GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=thumbnail", actor=actor + ) + crop = client.call( + "GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=crop", actor=actor + ) + optimized = client.call( + "GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=optimized", actor=actor + ) + if jpeg_size(thumb["bytes"]) != (256, 160): + raise Failure("thumbnail size") + if jpeg_size(crop["bytes"]) != (128, 128): + raise Failure("crop size") + if optimized["bytes"][:4] != b"RIFF" or optimized["bytes"][8:12] != b"WEBP": + raise Failure("optimized is not webp") + if names["thumbnail"]["width"] != 256 or names["thumbnail"]["height"] != 160: + raise Failure("thumbnail metadata") + if names["crop"]["width"] != 128 or names["crop"]["height"] != 128: + raise Failure("crop metadata") + if names["optimized"]["width"] != 320 or names["optimized"]["height"] != 200: + raise Failure("optimized metadata") + if hashlib.sha256(thumb["bytes"]).hexdigest() != names["thumbnail"]["sha256"]: + raise Failure("thumbnail sha") + rows = state(ready["id"]) + version_rows = [row for row in rows["variants"] if row["pipeline_version"] == rows["pipeline_version"]] + if len(version_rows) != 4: + raise Failure("variant rows " + str(len(version_rows))) + return { + "pipeline_version": rows["pipeline_version"], + "variant_rows": len(version_rows), + "thumbnail": [names["thumbnail"]["width"], names["thumbnail"]["height"]], + "crop": [names["crop"]["width"], names["crop"]["height"]], + "optimized": [names["optimized"]["width"], names["optimized"]["height"]], + "original_sha256": hashlib.sha256(original).hexdigest(), + } + + +def upload(actor, chat): + att, content = client.attachment(actor, chat) + return att, content + + +results = [] + + +def scenario(name, fn): + started = time.monotonic() + try: + detail = fn() + results.append({"name": name, "ok": True, "seconds": round(time.monotonic() - started, 3), "detail": detail}) + except Exception as exc: + results.append({"name": name, "ok": False, "seconds": round(time.monotonic() - started, 3), "error": type(exc).__name__ + ": " + str(exc)}) + raise + + +client = Client(BASE, timeout=20) + + +def durability(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + blocked = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob) + text = client.message(alice, chat, "текст пока файл в очереди")["json"] + linked = client.message(alice, chat, "", attachments=[att["id"]])["json"] + before_api = depth() + if before_api < 1: + raise Failure("job was not in the durable queue") + restart_service("api", ["mtusi-lab5-api-1", "mtusi-lab5-api-2"]) + after_api = depth() + restart_service("rabbitmq", "mtusi-lab5-rabbitmq-1", 180) + after_rabbit = depth() + if after_api < 1 or after_rabbit < 1: + raise Failure("queue lost the job") + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + return { + "upload_status": att["status"], + "content_while_queued": blocked["status"], + "text_status": 201, + "text_id": text["id"], + "linked_id": linked["id"], + "queue_before_api_restart": before_api, + "queue_after_api_restart": after_api, + "queue_after_rabbit_restart": after_rabbit, + "picture": picture, + } + + +def outbox_gap(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + arm("skip_inline_publish", None) + before = depth() + att, content = upload(alice, chat) + info = state(att["id"]) + queued = depth() + if info["unpublished"] != 1 or queued != before: + raise Failure("outbox was published early") + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + return {"unpublished_before_worker": info["unpublished"], "queue_depth": queued, "picture": picture} + + +def duplicate(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + info = state(att["id"]) + job_id = info["jobs"][0]["id"] + published = inside({"action": "republish", "job_id": job_id}) + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + return {"republish": published["ok"], "picture": picture} + + +def reorder(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + info = state(att["id"]) + job_id = info["jobs"][0]["id"] + inside({"action": "republish", "job_id": job_id, "attempt": 2}) + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + after = state(att["id"]) + if after["jobs"][0]["attempt"] != 1 or after["status"] != "ready": + raise Failure("reorder changed the job") + return {"attempt": after["jobs"][0]["attempt"], "picture": picture} + + +def stale_version(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + info = state(att["id"]) + old_job = info["jobs"][0]["id"] + bumped = inside({"action": "bump", "attachment_id": att["id"]}) + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + before = check_picture(bob, ready, content) + thumb_before = client.call( + "GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob + )["bytes"] + inside({"action": "republish", "job_id": old_job, "attempt": 1}) + time.sleep(2) + ready2 = client.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"] + thumb_after = client.call( + "GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob + )["bytes"] + after = state(att["id"]) + old = [job for job in after["jobs"] if job["id"] == old_job][0] + if ready2["status"] != "ready" or after["pipeline_version"] != 2: + raise Failure("new version was overwritten") + if old["status"] != "superseded": + raise Failure("old job was not superseded") + if thumb_before != thumb_after: + raise Failure("old job changed thumbnail") + return { + "bump": bumped["result"], + "old_job_status": old["status"], + "old_error_code": old["error_code"], + "attachment_error_code": after["error_code"], + "picture": before, + "thumbnail_unchanged": True, + } + + +def crash(kind): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + arm(kind, att["id"]) + start_worker() + born = inspect("mtusi-lab5-worker-1", "{{.State.StartedAt}}") + ready = wait_status(bob, att["id"], "ready", 120) + picture = check_picture(bob, ready, content) + restarted = inspect("mtusi-lab5-worker-1", "{{.State.StartedAt}}") + if born == restarted: + raise Failure("worker did not exit") + return {"started_at": born, "restarted_at": restarted, "picture": picture} + + +def transient(): + alice, bob, _eve, chat = client.bootstrap() + stop_worker() + att, _content = upload(alice, chat) + arm("transient_once", att["id"]) + start_worker() + wait_status(bob, att["id"], "ready", 90) + info = state(att["id"]) + version_rows = [row for row in info["variants"] if row["pipeline_version"] == info["pipeline_version"]] + if info["jobs"][0]["attempt"] != 2 or "retry" not in info["events"] or len(version_rows) != 4: + raise Failure("transient retry did not keep one result") + return { + "attempt": info["jobs"][0]["attempt"], + "events": info["events"], + "variant_rows": len(version_rows), + "status": info["status"], + } + + +def permanent_and_replay(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + corrupted = inside({"action": "corrupt", "attachment_id": att["id"]}) + start_worker() + failed = wait_status(bob, att["id"], "failed", 90) + info = state(att["id"]) + restored = inside({"action": "restore", "attachment_id": att["id"]}) + replayed = inside({"action": "replay", "job_id": info["jobs"][0]["id"]}) + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + again = inside({"action": "replay", "job_id": info["jobs"][0]["id"]}) + after = state(att["id"]) + version_rows = [row for row in after["variants"] if row["pipeline_version"] == after["pipeline_version"]] + if failed["error_code"] != "invalid_image" or info["jobs"][0]["attempt"] != 1: + raise Failure("permanent error was retried") + if replayed["result"].get("result") != "queued" or again["result"].get("result") != "already_ready": + raise Failure("replay result") + if len(version_rows) != 4 or restored["restored_sha256"] != corrupted["original_sha256"]: + raise Failure("replay did not keep one result") + return { + "corrupt_sha256": corrupted["original_sha256"], + "failed_code": failed["error_code"], + "attempt_at_fail": info["jobs"][0]["attempt"], + "restored_sha256": restored["restored_sha256"], + "replay": replayed["result"], + "second_replay": again["result"], + "variant_rows": len(version_rows), + "picture": picture, + } + + +def lease(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + arm("skip_inline_publish", None) + before = depth() + att, content = upload(alice, chat) + info = state(att["id"]) + if info["unpublished"] != 1 or depth() != before: + raise Failure("lease setup published the job") + job_id = info["jobs"][0]["id"] + psql( + "UPDATE attachment_jobs SET status = 'processing', lease_owner = 'dead', " + "lease_until = now() - interval '2 minutes', updated_at = now() " + "WHERE id = '{id}'::uuid".format(id=job_id) + ) + psql( + "UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = '{id}'::uuid".format( + id=att["id"] + ) + ) + psql("DELETE FROM outbox WHERE job_id = '{id}'::uuid AND published_at IS NULL".format(id=job_id)) + started = time.monotonic() + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + elapsed = round(time.monotonic() - started, 3) + picture = check_picture(bob, ready, content) + info = state(att["id"]) + if "lease_expired" not in info["events"] or info["jobs"][0]["attempt"] != 1: + raise Failure("expired lease was not resumed") + return { + "seconds_until_ready": elapsed, + "events": info["events"], + "attempt": info["jobs"][0]["attempt"], + "picture": picture, + } + + +def http_status(url): + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=10) as response: + return response.status, response.read(4096).decode("utf-8", "replace") + except urllib.error.HTTPError as error: + return error.code, error.read(4096).decode("utf-8", "replace") + + +def leaked(text): + lowered = text.lower() + return [ + word for word in ( + "redis", "postgres", "minio", "password", "select", "rabbit", "amqp", + "secret", "database", "bucket", "messenger-attachments", + ) + if word in lowered + ] + + +def ready_503(): + compose("stop", "redis") + time.sleep(2) + lb_status, lb_body = http_status(BASE + "/health/ready") + proc = run( + [ + "docker", "exec", "mtusi-lab5-api-1", "python", "-c", + "import urllib.request,urllib.error\n" + "req=urllib.request.Request('http://127.0.0.1:8080/health/ready')\n" + "try:\n" + " r=urllib.request.urlopen(req, timeout=5)\n" + " print(r.status)\n" + " print(r.read(4096).decode())\n" + "except urllib.error.HTTPError as e:\n" + " print(e.code)\n" + " print(e.read(4096).decode())\n", + ], + timeout=20, + ) + lines = proc.stdout.decode("utf-8", "replace").splitlines() + api_status = int(lines[0]) + api_body = "\n".join(lines[1:]) + compose("start", "redis") + wait_health("mtusi-lab5-redis-1") + started = time.monotonic() + deadline = started + 40 + recovered = None + while time.monotonic() < deadline: + try: + recovered, _body = http_status(BASE + "/health/ready") + except (urllib.error.URLError, TimeoutError, OSError): + recovered = 0 + if recovered == 200: + break + time.sleep(0.5) + recovery_seconds = round(time.monotonic() - started, 3) + if lb_status != 503 or api_status != 503 or leaked(lb_body) or leaked(api_body): + raise Failure("ready 503 text") + if "Сервис временно недоступен" not in api_body or recovered != 200: + raise Failure("ready 503 text") + return { + "lb_status": lb_status, + "lb_body": lb_body, + "api_status": api_status, + "api_message": "Сервис временно недоступен", + "forbidden": [], + "recovered": recovered, + "recovery_seconds": recovery_seconds, + } + + +def main(): + wait_health("mtusi-lab5-api-1") + wait_health("mtusi-lab5-api-2") + psql("UPDATE fault_injections SET armed = false WHERE armed") + start_worker() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if depth() == 0 and queues().get("attachments.retry", 0) == 0: + break + time.sleep(0.5) + steps = [ + ("durability", durability), + ("outbox_gap", outbox_gap), + ("duplicate", duplicate), + ("reorder", reorder), + ("stale_version", stale_version), + ("crash_before_effect", lambda: crash("crash_before_effect")), + ("crash_after_objects", lambda: crash("crash_after_objects")), + ("crash_after_commit", lambda: crash("crash_after_commit")), + ("transient_once", transient), + ("permanent_replay", permanent_and_replay), + ("lease", lease), + ("ready_503", ready_503), + ] + if len(sys.argv) > 1 and sys.argv[1] == "resume" and EVIDENCE.is_file(): + done = [ + item for item in json.loads(EVIDENCE.read_text(encoding="utf-8"))["results"] if item.get("ok") + ] + results.extend(done) + steps = [item for item in steps if item[0] not in {row["name"] for row in done}] + failed = None + for name, fn in steps: + try: + scenario(name, fn) + except Exception as exc: + failed = exc + break + EVIDENCE.write_text(json.dumps({"results": results}, ensure_ascii=False, indent=2), encoding="utf-8") + if failed is not None: + print("FAIL " + type(failed).__name__ + ": " + str(failed), file=sys.stderr) + return 1 + print("PASS pipeline") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Failure as exc: + print("FAIL " + str(exc), file=sys.stderr) + raise SystemExit(1)