1 Commits
61 changed files with 7420 additions and 23 deletions
+2
View File
@@ -28,3 +28,5 @@ OTEL_SERVICE_NAME=messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
OTEL_TRACES_SAMPLER_ARG=1.0
POOL_MAX=10
+4
View File
@@ -0,0 +1,4 @@
*.sh text eol=lf
*.cfg text eol=lf
Dockerfile text eol=lf
*.sql text eol=lf
+28
View File
@@ -0,0 +1,28 @@
FROM python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONPATH=/app/src \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
PIP_DEFAULT_TIMEOUT=120
WORKDIR /app
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir --upgrade pip==25.0.1 \
&& pip install --no-cache-dir -r /app/requirements.txt
COPY migrations /app/migrations
COPY src /app/src
USER 10001
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=25s --retries=8 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"
CMD ["python", "/app/src/main.py"]
+120 -17
View File
@@ -1,37 +1,140 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Орлова Владислава, БПИ2301
- Целевая оценка: 5. Трек 2, SLO и обнаружение инцидентов
- Commit/PR: ещё не публиковала
- Источники и помощь: задание и контракт этой лабораторной, рабочий стенд лабораторной 5, документация OpenTelemetry Collector, W3C Trace Context, Prometheus histograms и Grafana provisioning. При написании кода и отчёта пользовалась помощью ИИ. Числа ниже сняты с моего запуска 16 ноября 2026, я их не округляла
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Engine и Python 3.9+. У меня Docker Engine 26.1.4 и Python на Windows. Программы `make` в системе нет, поэтому команды из Makefile я запускала через `python -X utf8`. Флаг нужен, чтобы Windows не читала JSON в кодировке cp1251. Публичные `.sh` Windows не запускает как программу, их я запускала через WSL.
```sh
python -X utf8 scripts/bootstrap.py
docker compose --env-file .env up -d --build --scale api=2
python -X utf8 scripts/wait_ready.py
python -X utf8 scripts/check_contract.py
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
```
Проверки сохранности и остановки реплики:
```sh
wsl -e bash -lc "cd '/mnt/c/Git/высоконагруженные приложения/Lab6' && python3 -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080"
wsl -e bash -lc "cd '/mnt/c/Git/высоконагруженные приложения/Lab6' && python3 -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080"
```
`bootstrap.py` один раз создаёт `secrets/` и `.env`. Пароли в консоль не печатаются. Повторный запуск уже созданные файлы не перезаписывает. В git эти файлы не входят.
Остановить стенд, не удаляя данные: `docker compose --env-file .env stop`. Удалить учебные данные вместе с volume: `docker compose --env-file .env down -v`.
Имя проекта Compose — `mtusi-lab6`. Соседняя папка лабораторной 5 из-за этого не пересекается со стендом. Наружу открыты `127.0.0.1:8080` у балансировщика, `127.0.0.1:3000` у Grafana и `127.0.0.1:9090` у Prometheus. OTLP наружу не публикую.
Образы, которые реально поднимаются:
| Роль | Образ |
| --- | --- |
| API и worker | свой Dockerfile, база `python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db` |
| Postgres | `postgres:16.10-bookworm@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74` |
| Redis | `redis:7.4.2-bookworm@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9` |
| RabbitMQ | `rabbitmq:3.13.7-alpine@sha256:d7af1c87c5f1eda13fcfca06db452bf3aeab6619fc3358b68535c0c02c4e52bc` |
| Балансировщик | `haproxy:2.8.14@sha256:2344d1efbf7a932c7a3d49b60748c28af81c70fd11fcadd983e2be1f0376a69d` |
| MinIO | `pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372` |
| OTel Collector | `otel/opentelemetry-collector-contrib:0.114.0@sha256:37fa87091cfaaec7234a27e4e395a40c31c2bfaea97a349a4afef6d9e9681197` |
| Prometheus | `prom/prometheus:v2.54.1@sha256:f6639335d34a77d9d9db382b92eeb7fc00934be8eae81dbc03b31cfe90411a94` |
| Loki | `grafana/loki:3.2.1@sha256:09a53b4a4ff81ffcd8f13886df19d33fac7a8d3aaf952e3c7e66cbade5b2fc31` |
| Tempo | `grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50` |
| Grafana | `grafana/grafana:11.2.2@sha256:d5133220d770aba5cb655147b619fa8770b90f41d8489a821d33b1cd34d16f89` |
Сервер внутри образа MinIO отвечает `minio version RELEASE.2026-08-04T00-00-00Z`. Официальный `minio/minio` с Docker Hub больше не скачивается, а `quay.io/minio/minio` без логина отвечает `unauthorized`. Поэтому стенд закреплён на публичном образе `pgsty/minio` с digest. Это сервер MinIO с тем же протоколом S3.
Grafana на учебном стенде: пользователь `admin`, пароль `labadmin`. Для просмотра dashboard достаточно анонимного Viewer.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Из лабораторной 5 я перенесла весь бизнес-стенд: две реплики API, HAProxy, Postgres, Redis-сессии, MinIO, RabbitMQ, worker, outbox, идемпотентность и старые текстовые правила. Чужой объект по-прежнему даёт 404. UUID в разном регистре — один идентификатор, в базе он в нижнем регистре. Миграции применяет одноразовый контейнер `migrate`.
Новое в этой работе: API и worker отдают OTLP в Collector. Collector пишет metrics в Prometheus scrape endpoint `:8889`, traces в Tempo, logs в Loki. Grafana поднимается с provisioning datasources и dashboard `Messenger observability`.
Соответствие сигналов `contracts/observability.md` и имён в Prometheus:
| Сигнал | Имя в приложении | Имя в Prometheus | Единицы |
| --- | --- | --- | --- |
| HTTP requests | `messenger.http.requests` | `messenger_http_requests_total` | 1 |
| HTTP duration | `messenger.http.duration` | `messenger_http_duration_seconds` | секунды |
| Jobs processed | `messenger.jobs.processed` | `messenger_jobs_processed_total` | 1 |
| Job duration | `messenger.jobs.duration` | `messenger_jobs_duration_seconds` | секунды |
| Queue depth | `messenger.queue.depth` | `messenger_queue_depth` | 1 |
| Oldest job age | `messenger.queue.oldest_job_age` | `messenger_queue_oldest_job_age_seconds` | секунды |
| DB pool | `messenger.db.pool.used/waiting` | `messenger_db_pool_used_ratio` / `messenger_db_pool_waiting_ratio` | 1; суффикс `_ratio` ставит Prometheus exporter из-за unit=`1` |
| Worker saturation | `messenger.worker.busy/capacity` | `messenger_worker_busy_ratio` / `messenger_worker_capacity_ratio` | 1 |
| Time to ready | `messenger.attachment.time_to_ready` | `messenger_attachment_time_to_ready_seconds` | секунды |
Labels метрик низкокардинальные: method, route template, status class, job type/outcome, queue name. `user_id` / `chat_id` / `message_id` в labels не использую.
W3C trace context кладётся в поля `traceparent`/`tracestate` сообщения задания из `contracts/jobs.schema.json`. Worker открывает consumer span как child этого контекста. На прогоне upload trace `5a52413dfcd7d8afab436ca90e59877c` имел root `POST /api/v1/chats/{chat_id}/attachments` у `messenger-api` и продолжение обработки у worker в том же trace.
Назначение хранилищ:
- Prometheus — time series метрик и alert rules
- Loki — структурные логи с `trace_id`/`span_id`
- Tempo — трассы API и worker
- Grafana — запросы и dashboard, сама данные трёх сигналов не хранит
- Collector — приём OTLP, batch, memory limiter; не долговременная БД
Collector: `memory_limiter` 256 MiB, batch, для Tempo/Loki очередь экспорта 256 и `max_elapsed_time=30s`. Metrics идут через pull Prometheus exporter, без бесконечной очереди в backend. Когда я останавливала Tempo/Loki/Prometheus, upload вернул 202, текстовое сообщение 201, ready остался 200, память Collector была `72.56MiB` до и `72.6MiB` после.
Переменные из `.env.example`: `SESSION_STORE` в примере `postgres`, в compose у API стоит `redis`. `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318`, у worker отдельный `OTEL_SERVICE_NAME=messenger-worker`.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Машина: Intel Core i5-10300H, 8 логических процессоров, 17008717824 байт памяти. Дата прогона 16 ноября 2026.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий | Команда и что я увидела | Файл |
| --- | --- | --- |
| Контракт на диске | `python -X utf8 scripts/check_contract.py` → PASS | — |
| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080` → `PASS: lab6 public API scenario...` | `evidence/smoke.txt` |
| Сохранность | `tests/resilience.py --mode persistence` через WSL, PASS. StartedAt сменился у postgres, redis, minio, rabbitmq, prometheus, loki, tempo и обеих API | `evidence/persistence.txt`, `evidence/persistence-before.txt`, `evidence/persistence-after.txt` |
| Остановка реплики | failover через WSL: 2 реплики до отказа, successful_writes=91, failed_attempts=1, first_recovery_seconds=1.307 | `evidence/failover.txt` |
| Негативные правила | `python -X utf8 student-tests/run_negative.py` → PASS negative | `evidence/negative.json` |
| Метрики/логи/traces | В Prometheus есть `messenger_http_*`, `messenger_jobs_*`, queue/pool/worker. В Loki есть `{service_name="messenger-api"} |= "http_request"` с `trace_id`. В Tempo upload и worker в одном trace | `evidence/observability.json`, `evidence/traces-queue.txt` |
| Инцидент worker stop | Возраст job вырос до 77.8 с, alert `GrowingQueueLag` firing через 99.1 с | `evidence/observability.json` |
| Инцидент DB stop | Прямые 503 в API, `HighHttpErrorRate` firing, `ErrorBudgetBurnFast` firing | `evidence/observability.json` |
| Telemetry backend down | API жив, RAM Collector не выросла | `evidence/observability.json` |
| Трек 2 burn-rate | Краткий всплеск 25 с: HighHttpErrorRate через 83.1 с, BurnFast не зажёгся. Длинная деградация 360 с: BurnSlow и BurnFast через 454.9 с; error_ratio_5m=1.0, error_ratio_30m≈0.601 | `evidence/observability.json` |
Публичные файлы в `contracts/` и `tests/` я не меняла.
p50/p95/p99 на dashboard считаются через `histogram_quantile` по bucket-метрикам, не через среднее. Запросы лежат в `infra/grafana/dashboards/messenger.json`.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
### SLO
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
SLI API: доля HTTP 5xx на бизнес-маршрутах, health исключён.
SLO: error ratio ≤ 1% на учебном бюджете.
Окна burn-rate: fast = 14.4× на 5м и 30м, slow = 6× на 30м. Alert rules в `infra/prometheus/alerts.yml`, runbooks в `evidence/runbooks/`.
Гипотеза трека 2: короткий всплеск должен зажечь обычный high-error alert и не обязан зажигать fast burn на двух окнах; длительная деградация должна зажечь slow burn.
Что наблюдали:
| Воздействие | Ожидание | Наблюдение | Восстановление |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| stop worker + upload | рост oldest age, GrowingQueueLag | age 31→78 с, firing через 99.1 с | start worker, файл доходит до ready |
| stop postgres + прямые запросы в API | 5xx, HighHttpErrorRate | 5xx counter 5→18, HighHttpErrorRate и BurnFast firing | start postgres, ready снова 200 |
| stop tempo/loki/prometheus | API жив, RAM Collector ограничена | upload 202, message 201, RAM 72.56→72.6 MiB | start backends |
| краткий postgres outage 25 с | HighHttpErrorRate, BurnFast может не зажечься | HighHttpErrorRate через 83.1 с, BurnFast = null | resolved после восстановления |
| длинный postgres outage 360 с | BurnSlow | BurnSlow и BurnFast через 454.9 с | resolved после восстановления |
Ложное срабатывание BurnFast на коротком всплеске я не получила. Это ожидаемо: на окне 30м краткий отказ не держит 14.4× бюджета.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Стенд переживает остановку одной реплики API, рестарт хранилищ с сохранёнными volume, остановку worker на время приёма файла и краткую недоступность telemetry backends. Выключение компьютера и поломка диска данные не сохраняют. Postgres, Redis, MinIO, RabbitMQ, HAProxy и каждое telemetry-хранилище в стенде по одному экземпляру.
Если обе реплики API красные по ready, клиент через HAProxy видит английскую HTML-страницу 503, а не JSON API. Поэтому для инцидента с Postgres я снимала бизнес-5xx прямым запросом внутрь контейнера API и честно написала об этом.
Возраст очереди публикуют и API, и worker. Alert смотрит `max(...)`, иначе после остановки worker осталась бы устаревшая нулевая серия.
Прямую загрузку по временной ссылке я не добавляла. Трек 1 про цену наблюдаемости не заявляю: отдельных измерений latency/CPU/RAM off/on у меня для этой работы нет, и я их не придумывала.
+425
View File
@@ -0,0 +1,425 @@
name: mtusi-lab6
services:
postgres:
image: postgres:16.10-bookworm@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
command:
- postgres
- -c
- fsync=on
- -c
- synchronous_commit=on
- -c
- full_page_writes=on
- -c
- max_connections=80
- -c
- shared_buffers=64MB
volumes:
- postgres-data:/var/lib/postgresql/data
secrets:
- postgres_password
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 3s
timeout: 3s
retries: 20
start_period: 10s
cpus: "1.00"
mem_limit: 512m
restart: unless-stopped
networks:
- lab
redis:
image: redis:7.4.2-bookworm@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9
command:
- redis-server
- --appendonly
- "yes"
- --appendfsync
- everysec
- --dir
- /data
- --save
- "60"
- "1"
- --bind
- 0.0.0.0
- --protected-mode
- "no"
volumes:
- redis-data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 3s
timeout: 3s
retries: 20
start_period: 5s
cpus: "0.25"
mem_limit: 128m
restart: unless-stopped
networks:
- lab
rabbitmq:
image: rabbitmq:3.13.7-alpine@sha256:d7af1c87c5f1eda13fcfca06db452bf3aeab6619fc3358b68535c0c02c4e52bc
hostname: rabbitmq
entrypoint: ["/bin/sh", "/init/rabbit-entry.sh"]
volumes:
- rabbitmq-data:/var/lib/rabbitmq
- ./scripts/rabbit-entry.sh:/init/rabbit-entry.sh:ro
secrets:
- rabbit_user
- rabbit_password
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 10s
retries: 12
start_period: 180s
cpus: "1.00"
mem_limit: 1024m
restart: unless-stopped
networks:
- lab
minio:
image: pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372
command: ["server", "/data", "--console-address", "127.0.0.1:9001"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
volumes:
- minio-data:/data
secrets:
- minio_root_user
- minio_root_password
healthcheck:
test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"]
interval: 3s
timeout: 3s
retries: 20
start_period: 8s
cpus: "0.50"
mem_limit: 512m
restart: unless-stopped
networks:
- lab
minio-init:
image: pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372
entrypoint: ["/bin/sh", "/init/minio-init.sh"]
volumes:
- ./scripts/minio-init.sh:/init/minio-init.sh:ro
- ./infra/minio-api-policy.json:/init/policy.json:ro
secrets:
- minio_root_user
- minio_root_password
- s3_access_key
- s3_secret_key
depends_on:
minio:
condition: service_healthy
restart: "no"
networks:
- lab
migrate:
build: .
image: mtusi-lab6-api:local
command: ["python", "/app/src/migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
healthcheck:
disable: true
cpus: "0.50"
mem_limit: 256m
restart: "no"
networks:
- lab
api:
build: .
image: mtusi-lab6-api:local
environment:
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: redis
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
POOL_MAX: ${POOL_MAX:-10}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ENDPOINT: minio:9000
S3_BUCKET: messenger-attachments
S3_SECURE: "false"
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-120}
OTEL_SERVICE_NAME: messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
SERVICE_VERSION: lab6
secrets:
- database_url
- cache_url
- broker_url
- s3_access_key
- s3_secret_key
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
rabbitmq:
condition: service_healthy
minio:
condition: service_healthy
minio-init:
condition: service_completed_successfully
otel-collector:
condition: service_started
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 5s
timeout: 3s
retries: 12
start_period: 25s
init: true
stop_grace_period: 10s
cpus: "0.50"
mem_limit: 512m
deploy:
replicas: 2
restart: unless-stopped
networks:
- lab
worker:
build: .
image: mtusi-lab6-api:local
command: ["python", "/app/src/worker.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ENDPOINT: minio:9000
S3_BUCKET: messenger-attachments
S3_SECURE: "false"
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
OTEL_SERVICE_NAME: messenger-worker
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
SERVICE_VERSION: lab6
secrets:
- database_url
- broker_url
- s3_access_key
- s3_secret_key
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
minio:
condition: service_healthy
minio-init:
condition: service_completed_successfully
otel-collector:
condition: service_started
healthcheck:
disable: true
init: true
stop_grace_period: 20s
cpus: "1.00"
mem_limit: 768m
restart: unless-stopped
networks:
- lab
lb:
image: haproxy:2.8.14@sha256:2344d1efbf7a932c7a3d49b60748c28af81c70fd11fcadd983e2be1f0376a69d
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
depends_on:
api:
condition: service_healthy
cpus: "0.25"
mem_limit: 128m
restart: unless-stopped
networks:
- lab
otel-collector:
image: otel/opentelemetry-collector-contrib:0.114.0@sha256:37fa87091cfaaec7234a27e4e395a40c31c2bfaea97a349a4afef6d9e9681197
command: ["--config=/etc/otelcol/config.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
depends_on:
prometheus:
condition: service_started
loki:
condition: service_started
tempo:
condition: service_started
cpus: "0.50"
mem_limit: 384m
restart: unless-stopped
networks:
- lab
prometheus:
image: prom/prometheus:v2.54.1@sha256:f6639335d34a77d9d9db382b92eeb7fc00934be8eae81dbc03b31cfe90411a94
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=48h
- --web.enable-remote-write-receiver
- --web.enable-lifecycle
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
- prometheus-data:/prometheus
ports:
- "127.0.0.1:9090:9090"
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
cpus: "0.50"
mem_limit: 512m
restart: unless-stopped
networks:
- lab
loki:
image: grafana/loki:3.2.1@sha256:09a53b4a4ff81ffcd8f13886df19d33fac7a8d3aaf952e3c7e66cbade5b2fc31
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki-data:/loki
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"]
interval: 5s
timeout: 3s
retries: 12
start_period: 15s
cpus: "0.50"
mem_limit: 512m
restart: unless-stopped
networks:
- lab
tempo:
image: grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo-data:/var/tempo
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"]
interval: 5s
timeout: 3s
retries: 12
start_period: 15s
cpus: "0.50"
mem_limit: 512m
restart: unless-stopped
networks:
- lab
grafana:
image: grafana/grafana:11.2.2@sha256:d5133220d770aba5cb655147b619fa8770b90f41d8489a821d33b1cd34d16f89
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: labadmin
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
GF_SERVER_HTTP_PORT: "3000"
GF_PATHS_PROVISIONING: /etc/grafana/provisioning
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana-data:/var/lib/grafana
ports:
- "127.0.0.1:3000:3000"
depends_on:
prometheus:
condition: service_started
loki:
condition: service_started
tempo:
condition: service_started
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"]
interval: 5s
timeout: 3s
retries: 12
start_period: 20s
cpus: "0.50"
mem_limit: 384m
restart: unless-stopped
networks:
- lab
networks:
lab:
name: mtusi-lab6-net
volumes:
postgres-data:
redis-data:
minio-data:
rabbitmq-data:
prometheus-data:
loki-data:
tempo-data:
grafana-data:
secrets:
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
postgres_password:
file: ./secrets/postgres_password
minio_root_user:
file: ./secrets/minio_root_user
minio_root_password:
file: ./secrets/minio_root_password
s3_access_key:
file: ./secrets/s3_access_key
s3_secret_key:
file: ./secrets/s3_secret_key
rabbit_user:
file: ./secrets/rabbit_user
rabbit_password:
file: ./secrets/rabbit_password
broker_url:
file: ./secrets/broker_url
+10 -2
View File
@@ -1,5 +1,13 @@
# Доказательства
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
Здесь лежат обезличенные результаты прогонов лабораторной 6:
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
- `smoke.txt` — публичный smoke
- `persistence*.txt` — сохранность и StartedAt контейнеров
- `failover.txt` — остановка одной API-реплики
- `negative.json` — негативные проверки контракта
- `observability.json` — инциденты, alerts, burn-rate, telemetry down
- `traces-queue.txt` — связка upload→worker trace и GrowingQueueLag
- `runbooks/` — локальные runbook для alerts
Секреты, токены и тела сообщений сюда не кладу.
Binary file not shown.
Binary file not shown.
+74
View File
@@ -0,0 +1,74 @@
{
"command": "python -X utf8 student-tests/run_negative.py",
"result": "PASS negative",
"results": [
{
"name": "chat_uuid_case",
"status": 200
},
{
"name": "blank_text",
"status": 400
},
{
"name": "bad_json",
"status": 400
},
{
"name": "short_password",
"status": 400
},
{
"name": "omitted_equals_empty",
"status": 201
},
{
"name": "same_key_other_body",
"status": 409
},
{
"name": "no_second_row",
"status": 200
},
{
"name": "same_key_other_chat",
"status": 201
},
{
"name": "foreign_chat",
"status": 404
},
{
"name": "bad_cursor",
"status": 400
},
{
"name": "slash_name",
"status": 400
},
{
"name": "huge_pixels",
"status": 413
},
{
"name": "bad_png",
"status": 415
},
{
"name": "type_mismatch",
"status": 415
},
{
"name": "oversize",
"status": 413
},
{
"name": "unknown_attachment",
"status": 404
},
{
"name": "no_token",
"status": 401
}
]
}
+64
View File
@@ -0,0 +1,64 @@
{
"started_at": "2026-11-16T17:09:51.151551+00:00",
"finished_at": "2026-11-16T17:53:00+00:00",
"base_url": "http://127.0.0.1:8080",
"scenarios": {
"trace_link": {
"ok": true,
"note": "W3C: consumer span живёт в том же trace, что и POST upload",
"sample_trace_id": "5a52413dfcd7d8afab436ca90e59877c",
"root_service": "messenger-api",
"root_name": "POST /api/v1/chats/{chat_id}/attachments",
"request_id_example": "4a726eff-62a4-4d13-a0f6-3a1a8c7c274e",
"loki_http_request_sample": {
"event": "http_request",
"route": "/api/v1/users",
"status": 201,
"trace_id": "411d51400fd602a598e0bfc09cad04ed",
"span_id": "46aecc4198246684"
}
},
"worker_stop": {
"ok": true,
"attachment_id": "ee0819de-ef13-4d74-9841-4321054e2dff",
"fired_after_seconds": 99.1,
"age_samples": [
{"t": 52.9, "age": 31.374413},
{"t": 59.5, "age": 47.035699},
{"t": 72.6, "age": 57.191765},
{"t": 92.1, "age": 77.813526},
{"t": 98.2, "age": 77.813526}
],
"alert": "GrowingQueueLag firing"
},
"db_stop": {
"ok": true,
"http_5xx_before": 5.0,
"http_5xx_after": 18.0,
"high_error_seen_after_seconds": 0.9,
"alerts_seen": ["HighHttpErrorRate firing", "ErrorBudgetBurnFast firing", "ErrorBudgetBurnSlow pending"],
"note": "Бизнес-5xx снимала прямым запросом в контейнер API, потому что HAProxy при красном ready отдаёт свою HTML 503"
},
"telemetry_backend_down": {
"ok": true,
"upload_status": 202,
"ready_status": 200,
"message_status": 201,
"message_id": "76d951e4-1a0b-4be9-9217-22526922ce4c",
"otel_mem_before": "72.56MiB / 384MiB",
"otel_mem_after": "72.6MiB / 384MiB"
},
"burn_rate_spike": {
"ok": true,
"brief_spike_seconds": 25,
"brief_high_error_fired_after_seconds": 83.1,
"brief_burn_fast_fired_after_seconds": null,
"long_degradation_seconds": 360,
"long_burn_slow_fired_after_seconds": 454.9,
"long_burn_fast_fired_after_seconds": 454.9,
"error_ratio_5m": 1.0,
"error_ratio_30m": 0.6013241420811717,
"note": "Короткий всплеск зажёг HighHttpErrorRate и не зажёг BurnFast. Длинная деградация зажгла BurnSlow и BurnFast."
}
}
}
+13
View File
@@ -0,0 +1,13 @@
/mtusi-lab6-rabbitmq-1 2026-11-16T18:28:15.409788935Z
/mtusi-lab6-tempo-1 2026-11-16T18:28:15.402512463Z
/mtusi-lab6-loki-1 2026-11-16T18:28:15.558035972Z
/mtusi-lab6-prometheus-1 2026-11-16T18:28:15.558016399Z
/mtusi-lab6-worker-1 2026-11-16T18:27:16.239021708Z
/mtusi-lab6-api-1 2026-11-16T18:28:38.065212222Z
/mtusi-lab6-api-2 2026-11-16T18:28:37.997671947Z
/mtusi-lab6-lb-1 2026-11-16T17:36:12.433640676Z
/mtusi-lab6-grafana-1 2026-11-16T16:45:56.954366023Z
/mtusi-lab6-otel-collector-1 2026-11-16T17:19:04.876580907Z
/mtusi-lab6-minio-1 2026-11-16T18:28:02.310130635Z
/mtusi-lab6-postgres-1 2026-11-16T18:28:02.454244411Z
/mtusi-lab6-redis-1 2026-11-16T18:28:02.440606451Z
+13
View File
@@ -0,0 +1,13 @@
/mtusi-lab6-worker-1 2026-11-16T18:27:16.239021708Z
/mtusi-lab6-api-1 2026-11-16T18:27:16.110150646Z
/mtusi-lab6-api-2 2026-11-16T18:27:16.763621666Z
/mtusi-lab6-prometheus-1 2026-11-16T17:45:48.743893558Z
/mtusi-lab6-lb-1 2026-11-16T17:36:12.433640676Z
/mtusi-lab6-grafana-1 2026-11-16T16:45:56.954366023Z
/mtusi-lab6-otel-collector-1 2026-11-16T17:19:04.876580907Z
/mtusi-lab6-tempo-1 2026-11-16T17:18:58.629211805Z
/mtusi-lab6-rabbitmq-1 2026-11-16T16:45:56.111337038Z
/mtusi-lab6-loki-1 2026-11-16T17:18:57.765840011Z
/mtusi-lab6-minio-1 2026-11-16T16:43:54.21378518Z
/mtusi-lab6-postgres-1 2026-11-16T17:26:31.932858009Z
/mtusi-lab6-redis-1 2026-11-16T16:43:54.228473573Z
Binary file not shown.
Binary file not shown.
+14
View File
@@ -0,0 +1,14 @@
# Runbook: ErrorBudgetBurnFast / ErrorBudgetBurnSlow
SLO API: доля HTTP 5xx на бизнес-маршрутах ≤ 1% на скользящем окне 30 суток (учебный бюджет).
Burn-rate:
- fast: 14.4x бюджета одновременно на окнах 5м и 30м → короткий сильный всплеск
- slow: 6x бюджета на окне 30м → длительная деградация
Шаги:
1. Сравни `messenger:http_error_ratio:5m` и `messenger:http_error_ratio:30m` в Prometheus.
2. Fast без slow обычно значит краткий всплеск; slow значит проблема тянется.
3. Дальше действуй как в `high-error-rate.md`: trace → log → зависимость.
4. Зафиксируй время firing и resolved. Ложные срабатывания возможны на очень малом RPS из‑за деления на малый знаменатель.
+8
View File
@@ -0,0 +1,8 @@
# Runbook: HighHttpErrorRate
1. Открой dashboard `Messenger observability` в Grafana на `http://127.0.0.1:3000`.
2. Смотри панель «API error ratio» и «API latency». Если растут только 5xx, это не клиентские 4xx.
3. В Tempo найди свежие spans `messenger-api` со status ERROR. По `http.route` отдели бизнес-маршруты от `/health/*`.
4. По `trace_id` из лога Loki (`{service_name="messenger-api"} |= "http_request"`) открой связанный trace.
5. Если в spans много ошибок psycopg/timeout — смотри Postgres и панель DB pool. Если ошибки только на upload — смотри MinIO и worker.
6. После восстановления убедись, что alert перешёл в `resolved`, а error ratio снова ниже 5%.
+8
View File
@@ -0,0 +1,8 @@
# Runbook: GrowingQueueLag
1. Смотри панель «Oldest job age» и «Queue depth» для `attachments.process`.
2. Проверь контейнер worker: `docker compose --env-file .env ps worker`.
3. Если worker остановлен — это ожидаемый учебный отказ. Запусти worker и дождись уменьшения возраста job.
4. Если worker жив, открой Tempo spans `attachments.process consume` и ищи retry/failed.
5. В Loki `{service_name="messenger-worker"}` ищи `job_retry` / `job_failed`.
6. Alert должен уйти в `resolved`, когда oldest age снова ниже 30 с.
Binary file not shown.
Binary file not shown.
+279
View File
@@ -0,0 +1,279 @@
{
"annotations": { "list": [] },
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"title": "API RPS",
"type": "timeseries",
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[1m]))",
"legendFormat": "rps",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": { "drawStyle": "line", "fillOpacity": 10 }
}
}
},
{
"title": "API error ratio (5xx / all business)",
"type": "timeseries",
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "messenger:http_error_ratio:5m",
"legendFormat": "5m error ratio",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": { "drawStyle": "line", "fillOpacity": 10 }
}
}
},
{
"title": "API latency p50/p95/p99",
"type": "timeseries",
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "histogram_quantile(0.50, sum(rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])) by (le))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum(rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])) by (le))",
"legendFormat": "p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum(rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])) by (le))",
"legendFormat": "p99",
"refId": "C"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": { "drawStyle": "line", "fillOpacity": 5 }
}
}
},
{
"title": "Jobs ready / failed / retry",
"type": "timeseries",
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "sum by (job_outcome) (rate(messenger_jobs_processed_total[1m]))",
"legendFormat": "{{job_outcome}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": { "drawStyle": "line", "fillOpacity": 10 }
}
}
},
{
"title": "Job duration p50/p95/p99",
"type": "timeseries",
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "histogram_quantile(0.50, sum(rate(messenger_jobs_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum(rate(messenger_jobs_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum(rate(messenger_jobs_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p99",
"refId": "C"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": { "drawStyle": "line", "fillOpacity": 5 }
}
}
},
{
"title": "Time to ready p50/p95/p99",
"type": "timeseries",
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "histogram_quantile(0.50, sum(rate(messenger_attachment_time_to_ready_seconds_bucket[5m])) by (le))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum(rate(messenger_attachment_time_to_ready_seconds_bucket[5m])) by (le))",
"legendFormat": "p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum(rate(messenger_attachment_time_to_ready_seconds_bucket[5m])) by (le))",
"legendFormat": "p99",
"refId": "C"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": { "drawStyle": "line", "fillOpacity": 5 }
}
}
},
{
"title": "Queue depth",
"type": "timeseries",
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "messenger_queue_depth{messaging_destination_name=~\"attachments\\\\.process|attachments\\\\.retry\"}",
"legendFormat": "{{messaging_destination_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": { "drawStyle": "line", "fillOpacity": 10 }
}
}
},
{
"title": "Oldest job age",
"type": "timeseries",
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "max(messenger_queue_oldest_job_age_seconds{messaging_destination_name=\"attachments.process\"})",
"legendFormat": "oldest age",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": { "drawStyle": "line", "fillOpacity": 10 }
}
}
},
{
"title": "DB pool used / waiting",
"type": "timeseries",
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "messenger_db_pool_used_ratio",
"legendFormat": "used {{service_name}}",
"refId": "A"
},
{
"expr": "messenger_db_pool_waiting_ratio",
"legendFormat": "waiting {{service_name}}",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": { "drawStyle": "line", "fillOpacity": 5 }
}
}
},
{
"title": "Worker busy / capacity",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "messenger_worker_busy_ratio",
"legendFormat": "busy",
"refId": "A"
},
{
"expr": "messenger_worker_capacity_ratio",
"legendFormat": "capacity",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": { "drawStyle": "line", "fillOpacity": 5 }
}
}
},
{
"title": "Process RSS",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"expr": "messenger_process_rss_bytes",
"legendFormat": "{{service_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"custom": { "drawStyle": "line", "fillOpacity": 5 }
}
}
},
{
"title": "API / worker logs",
"type": "logs",
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 32 },
"datasource": { "type": "loki", "uid": "loki" },
"targets": [
{
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
"refId": "A"
}
]
}
],
"schemaVersion": 39,
"tags": ["messenger", "lab6"],
"templating": { "list": [] },
"time": { "from": "now-30m", "to": "now" },
"timezone": "browser",
"title": "Messenger observability",
"uid": "messenger-lab6",
"version": 1
}
@@ -0,0 +1,13 @@
apiVersion: 1
providers:
- name: messenger
orgId: 1
folder: Messenger
type: file
disableDeletion: false
updateIntervalSeconds: 30
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
foldersFromFilesStructure: false
@@ -0,0 +1,45 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
uid: prometheus
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
timeInterval: 15s
- name: Loki
type: loki
access: proxy
uid: loki
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- datasourceUid: tempo
matcherRegex: '"trace_id":"([a-f0-9]+)"'
name: TraceID
url: "$${__value.raw}"
urlDisplayLabel: View Trace
- name: Tempo
type: tempo
access: proxy
uid: tempo
url: http://tempo:3200
editable: false
jsonData:
tracesToLogs:
datasourceUid: loki
filterByTraceID: true
filterBySpanID: false
mapTagNamesEnabled: false
serviceMap:
datasourceUid: prometheus
nodeGraph:
enabled: true
lokiSearch:
datasourceUid: loki
+47
View File
@@ -0,0 +1,47 @@
global
maxconn 256
tune.bufsize 65536
tune.maxrewrite 8192
defaults
mode http
timeout connect 500ms
timeout client 60s
timeout server 60s
timeout http-request 60s
timeout http-keep-alive 2s
timeout queue 2s
timeout check 800ms
option redispatch
option dontlognull
retries 2
retry-on conn-failure empty-response 502 503 504
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold other 1s
hold refused 1s
hold nx 1s
hold timeout 1s
hold obsolete 1s
accepted_payload_size 8192
frontend web
bind *:8080
default_backend api
backend api
balance roundrobin
# Короткий JSON сообщения можно повторить. Загрузку файла не повторяем:
# повтор создал бы второй объект. Тело загрузки не буферизуется целиком.
acl upload method POST path_reg -i ^/api/v1/chats/[0-9a-fA-F-]+/attachments$
http-request wait-for-body time 2s if METH_POST !upload
http-request disable-l7-retry if upload
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
server-template api 8 api:8080 check inter 500ms fall 2 rise 2 resolvers docker init-addr none resolve-prefer ipv4 on-marked-down shutdown-sessions
+49
View File
@@ -0,0 +1,49 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
storage_config:
filesystem:
directory: /loki/chunks
tsdb_shipper:
active_index_directory: /loki/tsdb-index
cache_location: /loki/tsdb-cache
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
ingestion_rate_mb: 16
ingestion_burst_size_mb: 32
allow_structured_metadata: true
volume_enabled: true
ruler:
alertmanager_url: http://localhost:9093
analytics:
reporting_enabled: false
+19
View File
@@ -0,0 +1,19 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
},
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
+6 -4
View File
@@ -1,7 +1,9 @@
# Конфигурация наблюдаемости — реализовать
# OpenTelemetry Collector
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
Файл `collector.yaml` принимает OTLP HTTP/gRPC от API и worker, ограничивает память (`memory_limiter`), батчит экспорт и отправляет:
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
- traces → Tempo (`otlphttp/tempo`)
- metrics → Prometheus remote write
- logs → Loki OTLP (`/otlp`)
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
Очереди экспортеров ограничены (`queue_size: 256`). При недоступном backend retry ограничен `max_elapsed_time: 30s`, дальше сигнал отбрасывается. Это намеренно: приложение продолжает принимать сообщения, RAM Collector не растёт бесконечно.
+80
View File
@@ -0,0 +1,80 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
grpc:
endpoint: 0.0.0.0:4317
processors:
memory_limiter:
check_interval: 1s
limit_mib: 256
spike_limit_mib: 64
batch:
send_batch_size: 256
send_batch_max_size: 512
timeout: 2s
resource:
attributes:
- key: deployment.environment.name
value: lab
action: upsert
exporters:
otlphttp/tempo:
endpoint: http://tempo:4318
tls:
insecure: true
sending_queue:
enabled: true
num_consumers: 2
queue_size: 256
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 30s
# Pull-модель: Collector держит только последние значения в памяти, без бесконечной очереди.
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
enable_open_metrics: true
otlphttp/loki:
endpoint: http://loki:3100/otlp
tls:
insecure: true
sending_queue:
enabled: true
num_consumers: 2
queue_size: 256
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 30s
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions: [health_check]
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch, resource]
exporters: [otlphttp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch, resource]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch, resource]
exporters: [otlphttp/loki]
telemetry:
metrics:
level: basic
address: 0.0.0.0:8888
+88
View File
@@ -0,0 +1,88 @@
groups:
- name: messenger-slo
rules:
- record: messenger:http_requests:rate5m
expr: sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[5m])) or vector(0)
- record: messenger:http_5xx:rate5m
expr: sum(rate(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[5m])) or vector(0)
- record: messenger:http_error_ratio:5m
expr: |
(
messenger:http_5xx:rate5m
/
clamp_min(messenger:http_requests:rate5m, 0.001)
)
- record: messenger:http_requests:rate30m
expr: sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[30m])) or vector(0)
- record: messenger:http_5xx:rate30m
expr: sum(rate(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[30m])) or vector(0)
- record: messenger:http_error_ratio:30m
expr: |
(
messenger:http_5xx:rate30m
/
clamp_min(messenger:http_requests:rate30m, 0.001)
)
- record: messenger:http_requests:rate1h
expr: sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[1h])) or vector(0)
- record: messenger:http_5xx:rate1h
expr: sum(rate(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[1h])) or vector(0)
- record: messenger:http_error_ratio:1h
expr: |
(
messenger:http_5xx:rate1h
/
clamp_min(messenger:http_requests:rate1h, 0.001)
)
- name: messenger-alerts
rules:
- alert: HighHttpErrorRate
expr: messenger:http_error_ratio:5m > 0.05
for: 1m
labels:
severity: page
annotations:
summary: "Доля HTTP 5xx выше 5% за 5 минут"
description: "Текущая доля {{ $value | humanizePercentage }}. Runbook: evidence/runbooks/high-error-rate.md"
runbook_url: "evidence/runbooks/high-error-rate.md"
- alert: GrowingQueueLag
expr: max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"}) > 30
for: 30s
labels:
severity: ticket
annotations:
summary: "Возраст старейшей job в attachments.process выше 30 с"
description: "Возраст {{ $value }} с. Runbook: evidence/runbooks/queue-lag.md"
runbook_url: "evidence/runbooks/queue-lag.md"
- alert: ErrorBudgetBurnFast
expr: |
(
messenger:http_error_ratio:5m > (14.4 * 0.01)
and
messenger:http_error_ratio:30m > (14.4 * 0.01)
)
for: 2m
labels:
severity: page
burn_window: fast
annotations:
summary: "Быстрый burn rate ошибки SLO (14.4x на 5м и 30м)"
description: "Кратковременный всплеск 5xx. Runbook: evidence/runbooks/burn-rate.md"
runbook_url: "evidence/runbooks/burn-rate.md"
- alert: ErrorBudgetBurnSlow
expr: |
(
messenger:http_error_ratio:30m > (6 * 0.01)
)
for: 5m
labels:
severity: ticket
burn_window: slow
annotations:
summary: "Медленный burn rate ошибки SLO (6x на 30м)"
description: "Длительная деградация. Runbook: evidence/runbooks/burn-rate.md"
runbook_url: "evidence/runbooks/burn-rate.md"
+18
View File
@@ -0,0 +1,18 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
rule_files:
- /etc/prometheus/alerts.yml
scrape_configs:
- job_name: prometheus
static_configs:
- targets: ["localhost:9090"]
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8888"]
- job_name: messenger-otel
metrics_path: /metrics
static_configs:
- targets: ["otel-collector:8889"]
+47
View File
@@ -0,0 +1,47 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
grpc:
endpoint: 0.0.0.0:4317
ingester:
trace_idle_period: 10s
max_block_bytes: 1048576
max_block_duration: 5m
compactor:
compaction:
block_retention: 48h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
metrics_generator:
registry:
external_labels:
source: tempo
storage:
path: /var/tempo/generator/wal
overrides:
defaults:
metrics_generator:
processors: []
querier:
max_concurrent_queries: 20
query_frontend:
search:
max_duration: 1h
+48
View File
@@ -0,0 +1,48 @@
-- Пользователи, чаты, участники и сообщения.
-- Создание чата в приложении идёт одной транзакцией: строка chats и строки chat_members.
-- Индекс messages_history_idx совпадает с keyset-запросом истории:
-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
-- ORDER BY created_at, id
-- LIMIT $4
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL,
CONSTRAINT users_username_unique UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_nonblank CHECK (btrim(display_name) <> ''),
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL,
CONSTRAINT chats_title_nonblank CHECK (btrim(title) <> ''),
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id),
user_id uuid NOT NULL REFERENCES users (id),
ordinal smallint NOT NULL,
PRIMARY KEY (chat_id, user_id),
CONSTRAINT chat_members_ordinal_nonneg CHECK (ordinal >= 0)
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id),
sender_id uuid NOT NULL REFERENCES users (id),
body text NOT NULL,
created_at timestamptz NOT NULL,
CONSTRAINT messages_body_nonblank CHECK (btrim(body) <> ''),
CONSTRAINT messages_body_len CHECK (char_length(body) BETWEEN 1 AND 4000)
);
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
+19
View File
@@ -0,0 +1,19 @@
-- Результат успешного POST сообщения.
-- Область ключа: пользователь + метод + путь чата + Idempotency-Key.
-- Строка пишется в той же транзакции, что и messages, поэтому 5xx
-- не оставляет «занятый» ключ. Другое тело при живом ключе — отказ
-- на уровне приложения (хеш не совпал), без второй строки messages.
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users (id),
http_method text NOT NULL,
request_path text NOT NULL,
idem_key text NOT NULL,
request_hash text NOT NULL,
message_id uuid NOT NULL REFERENCES messages (id),
created_at timestamptz NOT NULL,
PRIMARY KEY (user_id, http_method, request_path, idem_key),
CONSTRAINT idempotency_method_post CHECK (http_method = 'POST'),
CONSTRAINT idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT idempotency_hash_len CHECK (char_length(request_hash) = 64)
);
+42
View File
@@ -0,0 +1,42 @@
-- Метаданные вложения. Байты лежат в приватном bucket, не в этой таблице.
-- Пустой текст допустим, если к сообщению прикреплено вложение. Пробелы по-прежнему нельзя.
ALTER TABLE messages DROP CONSTRAINT messages_body_nonblank;
ALTER TABLE messages DROP CONSTRAINT messages_body_len;
ALTER TABLE messages
ADD CONSTRAINT messages_body_len CHECK (char_length(body) <= 4000);
ALTER TABLE messages
ADD CONSTRAINT messages_body_nonblank CHECK (body = '' OR btrim(body) <> '');
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id),
owner_id uuid NOT NULL REFERENCES users (id),
filename text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
sha256 text NOT NULL,
object_key text NOT NULL UNIQUE,
status text NOT NULL DEFAULT 'ready',
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL,
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_content_type_chk CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size_chk CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha_chk CHECK (char_length(sha256) = 64),
CONSTRAINT attachments_status_chk CHECK (status = 'ready'),
CONSTRAINT attachments_key_chk CHECK (char_length(object_key) BETWEEN 1 AND 512)
);
CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id),
attachment_id uuid NOT NULL REFERENCES attachments (id),
ordinal integer NOT NULL,
PRIMARY KEY (message_id, attachment_id),
UNIQUE (message_id, ordinal),
CONSTRAINT message_attachments_ordinal_chk CHECK (ordinal >= 0 AND ordinal < 10)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
+108
View File
@@ -0,0 +1,108 @@
-- Задание на обработку, варианты и outbox.
-- Сообщение брокера не содержит байты и пароли: в outbox лежит только JSON задания.
-- Ключ эффекта — (attachment_id, pipeline_version, name). Повтор не создаёт вторую строку.
ALTER TABLE attachments DROP CONSTRAINT attachments_status_chk;
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
ALTER TABLE attachments ADD COLUMN error_code text;
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1;
ALTER TABLE attachments ADD COLUMN staging_key text;
ALTER TABLE attachments ADD COLUMN staging_deleted_at timestamptz;
ALTER TABLE attachments ADD CONSTRAINT attachments_status_chk CHECK (status IN ('queued', 'processing', 'ready', 'failed'));
ALTER TABLE attachments ADD CONSTRAINT attachments_pipeline_chk CHECK (pipeline_version >= 1);
ALTER TABLE attachments ADD CONSTRAINT attachments_error_chk CHECK (
(status = 'failed' AND error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted', 'processing_timeout'))
OR (status <> 'failed' AND error_code IS NULL)
);
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments (id),
pipeline_version integer NOT NULL,
status text NOT NULL,
attempt integer NOT NULL,
max_attempts integer NOT NULL,
lease_owner text,
lease_until timestamptz,
next_attempt_at timestamptz,
error_code text,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL,
CONSTRAINT attachment_jobs_status_chk CHECK (status IN ('queued', 'processing', 'ready', 'failed', 'superseded')),
CONSTRAINT attachment_jobs_attempt_chk CHECK (attempt >= 1 AND attempt <= max_attempts),
CONSTRAINT attachment_jobs_max_chk CHECK (max_attempts BETWEEN 1 AND 20),
CONSTRAINT attachment_jobs_version_chk CHECK (pipeline_version >= 1),
CONSTRAINT attachment_jobs_error_chk CHECK (
error_code IS NULL OR error_code IN (
'invalid_image', 'processing_failed', 'retry_exhausted', 'processing_timeout', 'superseded'
)
),
CONSTRAINT attachment_jobs_one_version UNIQUE (attachment_id, pipeline_version)
);
CREATE INDEX attachment_jobs_due_idx ON attachment_jobs (status, next_attempt_at);
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id),
pipeline_version integer NOT NULL,
name text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
sha256 text NOT NULL,
object_key text NOT NULL UNIQUE,
created_at timestamptz NOT NULL,
PRIMARY KEY (attachment_id, pipeline_version, name),
CONSTRAINT attachment_variants_name_chk CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')),
CONSTRAINT attachment_variants_type_chk CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')),
CONSTRAINT attachment_variants_size_chk CHECK (size_bytes >= 1),
CONSTRAINT attachment_variants_dim_chk CHECK (width >= 1 AND height >= 1),
CONSTRAINT attachment_variants_sha_chk CHECK (sha256 ~ '^[a-f0-9]{64}$')
);
CREATE TABLE outbox (
id uuid PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs (id),
attempt integer NOT NULL,
payload jsonb NOT NULL,
created_at timestamptz NOT NULL,
published_at timestamptz,
CONSTRAINT outbox_attempt_chk CHECK (attempt >= 1)
);
CREATE INDEX outbox_pending_idx ON outbox (created_at) WHERE published_at IS NULL;
CREATE TABLE job_events (
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id uuid NOT NULL,
attachment_id uuid NOT NULL,
pipeline_version integer NOT NULL,
kind text NOT NULL,
detail text,
created_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT job_events_kind_chk CHECK (char_length(kind) BETWEEN 1 AND 40),
CONSTRAINT job_events_detail_chk CHECK (detail IS NULL OR char_length(detail) <= 64)
);
CREATE TABLE fault_injections (
id uuid PRIMARY KEY,
kind text NOT NULL,
attachment_id uuid,
armed boolean NOT NULL DEFAULT true,
created_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT fault_injections_kind_chk CHECK (kind IN (
'skip_inline_publish', 'crash_before_effect', 'crash_after_objects',
'crash_after_commit', 'transient_once'
))
);
CREATE INDEX fault_injections_armed_idx ON fault_injections (kind, created_at) WHERE armed;
+14
View File
@@ -0,0 +1,14 @@
fastapi==0.115.12
uvicorn[standard]==0.34.2
pydantic==2.11.4
psycopg[binary]==3.2.6
psycopg-pool==3.2.4
redis==5.2.1
argon2-cffi==23.1.0
minio==7.2.15
pillow==11.1.0
pika==1.3.2
opentelemetry-api==1.27.0
opentelemetry-sdk==1.27.0
opentelemetry-exporter-otlp-proto-http==1.27.0
opentelemetry-semantic-conventions==0.48b0
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""Создать локальные .env и secrets для стенда. Пароль в консоль не печатается."""
import secrets
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRET_DIR = ROOT / "secrets"
PASSWORD_FILE = SECRET_DIR / "postgres_password"
DATABASE_FILE = SECRET_DIR / "database_url"
CACHE_FILE = SECRET_DIR / "cache_url"
MINIO_USER_FILE = SECRET_DIR / "minio_root_user"
MINIO_PASSWORD_FILE = SECRET_DIR / "minio_root_password"
S3_ACCESS_FILE = SECRET_DIR / "s3_access_key"
S3_SECRET_FILE = SECRET_DIR / "s3_secret_key"
RABBIT_USER_FILE = SECRET_DIR / "rabbit_user"
RABBIT_PASSWORD_FILE = SECRET_DIR / "rabbit_password"
BROKER_FILE = SECRET_DIR / "broker_url"
ENV_FILE = ROOT / ".env"
ENV_EXAMPLE = ROOT / ".env.example"
def main():
files = (PASSWORD_FILE, DATABASE_FILE, CACHE_FILE)
if any(path.exists() for path in files) and not all(path.exists() for path in files):
print("secrets/ заполнены не полностью, ничего не перезаписываю", file=sys.stderr)
return 1
SECRET_DIR.mkdir(exist_ok=True)
if all(path.exists() for path in files):
print("secrets already exist")
else:
password = secrets.token_hex(24)
PASSWORD_FILE.write_text(password + "\n", encoding="utf-8")
DATABASE_FILE.write_text(
"postgresql://messenger:{}@postgres:5432/messenger\n".format(password),
encoding="utf-8",
)
CACHE_FILE.write_text("redis://redis:6379/0\n", encoding="utf-8")
print("secrets created")
s3_files = (MINIO_USER_FILE, MINIO_PASSWORD_FILE, S3_ACCESS_FILE, S3_SECRET_FILE)
if any(path.exists() for path in s3_files) and not all(path.exists() for path in s3_files):
print("secrets/ для MinIO заполнены не полностью, ничего не перезаписываю", file=sys.stderr)
return 1
if all(path.exists() for path in s3_files):
print("s3 secrets already exist")
else:
MINIO_USER_FILE.write_text("labroot", encoding="utf-8")
MINIO_PASSWORD_FILE.write_text(secrets.token_hex(24), encoding="utf-8")
S3_ACCESS_FILE.write_text("api" + secrets.token_hex(8), encoding="utf-8")
S3_SECRET_FILE.write_text(secrets.token_hex(24), encoding="utf-8")
print("s3 secrets created")
broker_files = (RABBIT_USER_FILE, RABBIT_PASSWORD_FILE, BROKER_FILE)
if any(path.exists() for path in broker_files) and not all(path.exists() for path in broker_files):
print("secrets/ для брокера заполнены не полностью, ничего не перезаписываю", file=sys.stderr)
return 1
if all(path.exists() for path in broker_files):
print("broker secrets already exist")
else:
user = "lab"
password = secrets.token_hex(24)
RABBIT_USER_FILE.write_text(user, encoding="utf-8")
RABBIT_PASSWORD_FILE.write_text(password, encoding="utf-8")
BROKER_FILE.write_text(
"amqp://{}:{}@rabbitmq:5672/%2F\n".format(user, password),
encoding="utf-8",
)
print("broker secrets created")
if ENV_FILE.exists():
print(".env already exists")
else:
ENV_FILE.write_text(ENV_EXAMPLE.read_text(encoding="utf-8"), encoding="utf-8")
print(".env created")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
# Идемпотентно создаёт приватный bucket и отдельного пользователя API.
# Корневой пароль остаётся только у этого контейнера и у MinIO.
set -eu
ROOT_USER=$(tr -d '\r\n' < /run/secrets/minio_root_user)
ROOT_PW=$(tr -d '\r\n' < /run/secrets/minio_root_password)
ACCESS=$(tr -d '\r\n' < /run/secrets/s3_access_key)
SECRET=$(tr -d '\r\n' < /run/secrets/s3_secret_key)
attempt=0
until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PW" >/dev/null; do
attempt=$((attempt + 1))
if [ "$attempt" -gt 30 ]; then
echo "minio alias failed" >&2
exit 1
fi
sleep 1
done
mc mb --ignore-existing local/messenger-attachments >/dev/null
mc anonymous set none local/messenger-attachments >/dev/null
if ! mc admin user info local "$ACCESS" >/dev/null 2>&1; then
mc admin user add local "$ACCESS" "$SECRET" >/dev/null
fi
mc admin policy detach local messenger-api --user "$ACCESS" >/dev/null 2>&1 || true
mc admin policy rm local messenger-api >/dev/null 2>&1 || true
mc admin policy create local messenger-api /init/policy.json >/dev/null
mc admin policy attach local messenger-api --user "$ACCESS" >/dev/null
echo "minio bucket ready"
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
# Пароль читается из secret и попадает только в файл конфигурации внутри контейнера.
set -eu
user=$(tr -d '\r\n' < /run/secrets/rabbit_user)
pass=$(tr -d '\r\n' < /run/secrets/rabbit_password)
mkdir -p /tmp/rabbit-conf
printf 'default_user = %s\ndefault_pass = %s\nvm_memory_high_watermark.absolute = 768MB\n' "$user" "$pass" > /tmp/rabbit-conf/rabbitmq.conf
chmod 644 /tmp/rabbit-conf/rabbitmq.conf
export RABBITMQ_CONFIG_FILE=/tmp/rabbit-conf/rabbitmq
exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Рестарт API и всех хранилищ со состоянием. Volume не удаляем, миграции заново не заливаем.
# Балансировщик без своего состояния не трогаем: он продолжает слушать тот же порт.
# Telemetry-хранилища тоже рестартуем: у них свои volume с историей.
docker compose --env-file .env restart postgres redis minio rabbitmq prometheus loki tempo
docker compose --env-file .env up -d --wait --no-deps postgres redis minio rabbitmq prometheus loki tempo
docker compose --env-file .env restart api
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?}"
found=""
for id in $(docker ps -q \
--filter label=com.docker.compose.project=mtusi-lab6 \
--filter label=com.docker.compose.service=api); do
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
short=$(printf %s "$id" | cut -c1-12)
if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then
found=$id
break
fi
done
if [ -z "$found" ]; then
echo "api replica not found" >&2
exit 1
fi
# SIGTERM, затем процесс остаётся остановленным (restart: unless-stopped).
docker stop --time 5 "$found" >/dev/null
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env python3
"""Ждать GET /health/ready через балансировщик. Без токенов и тел ошибок в выводе."""
import os
import sys
import time
import urllib.error
import urllib.request
URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + "/health/ready"
def main():
deadline = time.monotonic() + 90
last = "not-started"
while time.monotonic() < deadline:
try:
response = urllib.request.urlopen(URL, timeout=2)
if response.status == 200 and response.headers.get("X-Instance-Id"):
print("ready")
return 0
last = str(response.status)
except (urllib.error.URLError, TimeoutError, OSError) as exc:
last = type(exc).__name__
time.sleep(1)
print("not ready: " + last, file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())
+228
View File
@@ -0,0 +1,228 @@
"""Проверка PNG/JPEG и доступ к приватному bucket. Ключ объекта назначает сервер."""
import io
import os
from datetime import datetime, timezone
from minio import Minio
from minio.error import S3Error
from PIL import Image, UnidentifiedImageError
from PIL.Image import DecompressionBombError
from errors import ApiError, DependencyDown
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
_client = None
def read_secret(path):
with open(path, encoding="utf-8") as handle:
return handle.read().strip()
def client():
global _client
if _client is None:
endpoint = os.environ.get("S3_ENDPOINT", "minio:9000").strip()
access = read_secret(os.environ["S3_ACCESS_KEY_FILE"])
secret = read_secret(os.environ["S3_SECRET_KEY_FILE"])
secure = os.environ.get("S3_SECURE", "false").strip().lower() == "true"
_client = Minio(endpoint, access_key=access, secret_key=secret, secure=secure)
return _client
def bucket_ready():
try:
return client().bucket_exists(BUCKET)
except Exception:
return False
def put_bytes(key, data, content_type, created_at):
stamp = created_at.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
try:
client().put_object(
BUCKET,
key,
io.BytesIO(data),
length=len(data),
content_type=content_type,
metadata={"created-at": stamp},
)
except S3Error as exc:
raise DependencyDown() from exc
def delete_quiet(key):
try:
client().remove_object(BUCKET, key)
return True
except Exception:
return False
def open_object(key):
try:
return client().get_object(BUCKET, key)
except S3Error as exc:
code = getattr(exc, "code", "")
if code in ("NoSuchKey", "NoSuchBucket"):
return None
raise DependencyDown() from exc
def object_key(chat_id, attachment_id):
return "chats/{}/attachments/{}/original".format(chat_id, attachment_id)
def staging_object_key(attachment_id):
return "staging/{}/original".format(attachment_id)
def variant_object_key(attachment_id, pipeline_version, name):
return "ready/{}/v{}/{}".format(attachment_id, int(pipeline_version), name)
def header_size(data, declared):
if declared == "image/png":
return _png_size(data)
if declared == "image/jpeg":
return _jpeg_size(data)
return None
def read_bytes(key):
body = open_object(key)
if body is None:
return None
try:
return body.read()
finally:
body.close()
body.release_conn()
def declared_type(header):
if not header:
return None
media = header.split(";", 1)[0].strip().lower()
if media in ("image/png", "image/jpeg"):
return media
return None
def _png_size(data):
if len(data) < 24 or data[:8] != b"\x89PNG\r\n\x1a\n" or data[12:16] != b"IHDR":
return None
width = int.from_bytes(data[16:20], "big")
height = int.from_bytes(data[20:24], "big")
return width, height
def _jpeg_size(data):
if len(data) < 4 or data[:2] != b"\xff\xd8":
return None
index = 2
limit = len(data)
while index + 8 < limit:
if data[index] != 0xFF:
return None
while index < limit and data[index] == 0xFF:
index += 1
if index >= limit:
return None
marker = data[index]
index += 1
if marker in (0xD8, 0xD9):
continue
if index + 2 > limit:
return None
segment = int.from_bytes(data[index:index + 2], "big")
if segment < 2 or index + segment > limit:
return None
if marker in (0xC0, 0xC1, 0xC2):
if segment < 7:
return None
height = int.from_bytes(data[index + 3:index + 5], "big")
width = int.from_bytes(data[index + 5:index + 7], "big")
return width, height
index += segment
return None
def _too_many_pixels(width, height):
if width <= 0 or height <= 0:
raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG")
if width > MAX_PIXELS or height > MAX_PIXELS or width * height > MAX_PIXELS:
raise ApiError(413, "payload_too_large", "В изображении больше 40 миллионов пикселей")
def inspect_image(data, declared):
if declared == "image/png":
size = _png_size(data)
elif declared == "image/jpeg":
size = _jpeg_size(data)
else:
size = None
detected = None
if data.startswith(b"\x89PNG\r\n\x1a\n"):
detected = "image/png"
elif data.startswith(b"\xff\xd8\xff"):
detected = "image/jpeg"
if size is None or detected != declared:
raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG")
_too_many_pixels(size[0], size[1])
try:
with Image.open(io.BytesIO(data)) as image:
image.load()
except DecompressionBombError as exc:
raise ApiError(413, "payload_too_large", "В изображении больше 40 миллионов пикселей") from exc
except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc:
raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG") from exc
return declared
def list_objects(prefix=""):
found = []
prefixes = (prefix,) if prefix else ("chats/", "staging/", "ready/")
try:
for one in prefixes:
found.extend(_list_prefix(one))
except S3Error as exc:
raise DependencyDown() from exc
return found
def _list_prefix(prefix):
found = []
try:
for item in client().list_objects(BUCKET, prefix=prefix, recursive=True):
if item.is_dir:
continue
stat = client().stat_object(BUCKET, item.object_name)
raw = None
if stat.metadata:
meta = {str(key).lower(): value for key, value in stat.metadata.items()}
raw = meta.get("x-amz-meta-created-at") or meta.get("created-at")
created = _parse_stamp(raw) if raw else None
if created is None and item.last_modified is not None:
created = item.last_modified
found.append({"key": item.object_name, "created_at": created})
except S3Error as exc:
raise DependencyDown() from exc
return found
def _parse_stamp(value):
text = value.strip().replace("Z", "+00:00")
try:
parsed = datetime.fromisoformat(text)
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed
+143
View File
@@ -0,0 +1,143 @@
"""RabbitMQ: publisher confirm при публикации, consumer ack отдельно в worker."""
import json
import os
from contextlib import contextmanager
import pika
from opentelemetry.trace import SpanKind
from psycopg.types.json import Json
from blobs import read_secret
from telemetry import start_span
PROCESS_QUEUE = "attachments.process"
RETRY_QUEUE = "attachments.retry"
FAILED_QUEUE = "attachments.failed"
def broker_url():
return read_secret(os.environ["BROKER_URL_FILE"])
def _parameters():
params = pika.URLParameters(broker_url())
params.heartbeat = 60
params.blocked_connection_timeout = 10
params.socket_timeout = 10
params.connection_attempts = 1
return params
def ensure_topology(channel):
channel.queue_declare(queue=PROCESS_QUEUE, durable=True)
channel.queue_declare(
queue=RETRY_QUEUE,
durable=True,
arguments={
"x-dead-letter-exchange": "",
"x-dead-letter-routing-key": PROCESS_QUEUE,
},
)
channel.queue_declare(queue=FAILED_QUEUE, durable=True)
@contextmanager
def broker_channel():
connection = pika.BlockingConnection(_parameters())
channel = connection.channel()
channel.confirm_delivery()
ensure_topology(channel)
try:
yield channel
finally:
try:
connection.close()
except Exception:
pass
def publish_raw(channel, queue, body, expiration_ms=None):
fields = {
"delivery_mode": 2,
"content_type": "application/json",
}
if expiration_ms is not None:
fields["expiration"] = str(int(expiration_ms))
with start_span(
"attachments.publish",
kind=SpanKind.PRODUCER,
attributes={
"messaging.system": "rabbitmq",
"messaging.destination.name": queue,
"messaging.operation": "publish",
},
):
channel.basic_publish(
exchange="",
routing_key=queue,
body=body,
properties=pika.BasicProperties(**fields),
mandatory=True,
)
def publish_pending(conn, only_job_id=None):
published = 0
while published < 20:
with conn.transaction():
conn.execute("SET LOCAL statement_timeout = '20s'")
params = []
sql = """
SELECT o.id, o.payload
FROM outbox o
JOIN attachment_jobs j ON j.id = o.job_id
WHERE o.published_at IS NULL
AND (j.next_attempt_at IS NULL OR j.next_attempt_at <= now())
"""
if only_job_id is not None:
sql += " AND o.job_id = %s"
params.append(only_job_id)
sql += """
ORDER BY o.created_at
FOR UPDATE OF o SKIP LOCKED
LIMIT 1
"""
row = conn.execute(sql, params).fetchone()
if row is None:
return published
body = json.dumps(
row["payload"],
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
with broker_channel() as channel:
publish_raw(channel, PROCESS_QUEUE, body)
conn.execute(
"UPDATE outbox SET published_at = now() WHERE id = %s AND published_at IS NULL",
(row["id"],),
)
published += 1
if only_job_id is not None:
return published
return published
def publish_delay(body, delay_ms):
with broker_channel() as channel:
publish_raw(channel, RETRY_QUEUE, body, expiration_ms=max(1000, int(delay_ms)))
def publish_failed(body):
with broker_channel() as channel:
publish_raw(channel, FAILED_QUEUE, body)
def queue_depth(queue):
with broker_channel() as channel:
declared = channel.queue_declare(queue=queue, passive=True)
return int(declared.method.message_count)
def dump_payload(payload):
return Json(payload)
+32
View File
@@ -0,0 +1,32 @@
class ApiError(Exception):
def __init__(self, status, code, message, headers=None):
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
class DependencyDown(Exception):
"""Postgres или Redis не ответили. Наружу это 503 без имени хранилища."""
class IdempotencyConflict(Exception):
"""Тот же Idempotency-Key уже зафиксирован с другим телом."""
class AttachmentRejected(Exception):
"""Вложение есть, но прикрепить его к сообщению нельзя."""
class AttachmentNotReady(Exception):
"""Байты ещё нельзя отдать: queued, processing или failed."""
def error_body(code, message, request_id):
return {
"error": {
"code": code,
"message": message,
"request_id": request_id,
}
}
+49
View File
@@ -0,0 +1,49 @@
"""Точечные сбои для проверки. Без строки в fault_injections код их не включает."""
import uuid
from psycopg.pq import TransactionStatus
def _idle(conn):
# Чтение выше могло открыть транзакцию. Её откат не теряет запись:
# сама запись начинается следующим transaction() и делает COMMIT.
if conn.info.transaction_status != TransactionStatus.IDLE:
conn.rollback()
def take_fault(conn, kind, attachment_id=None):
_idle(conn)
with conn.transaction():
row = conn.execute(
"""
UPDATE fault_injections AS target
SET armed = false
WHERE target.id = (
SELECT id
FROM fault_injections
WHERE armed
AND kind = %s
AND (attachment_id IS NULL OR attachment_id = %s)
ORDER BY created_at
FOR UPDATE SKIP LOCKED
LIMIT 1
)
RETURNING id
""",
(kind, attachment_id),
).fetchone()
return row is not None
def arm_fault(conn, kind, attachment_id=None):
fault_id = uuid.uuid4()
_idle(conn)
with conn.transaction():
conn.execute(
"""
INSERT INTO fault_injections (id, kind, attachment_id, armed)
VALUES (%s, %s, %s, true)
""",
(fault_id, kind, attachment_id),
)
return str(fault_id)
+129
View File
@@ -0,0 +1,129 @@
"""Варианты изображения. Оригинал не перекодируется."""
import io
from PIL import Image, ImageOps, UnidentifiedImageError
from PIL.Image import DecompressionBombError
from blobs import MAX_PIXELS, header_size
THUMB_BOX = 256
CROP_SIZE = 128
OPT_BOX = 1280
JPEG_QUALITY = 85
WEBP_QUALITY = 80
WEBP_METHOD = 4
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
class PermanentJobError(Exception):
def __init__(self, code):
super().__init__(code)
self.code = code
class TransientJobError(Exception):
def __init__(self, code):
super().__init__(code)
self.code = code
def _rgb(image):
if image.mode == "RGBA" or image.mode == "LA" or (image.mode == "P" and "transparency" in image.info):
rgba = image.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.getchannel("A"))
return background
if image.mode == "RGB":
return image
return image.convert("RGB")
def load_rgb(data, declared):
size = header_size(data, declared)
detected = None
if data.startswith(b"\x89PNG\r\n\x1a\n"):
detected = "image/png"
elif data.startswith(b"\xff\xd8\xff"):
detected = "image/jpeg"
if size is None or detected != declared:
raise PermanentJobError("invalid_image")
width, height = size
if width <= 0 or height <= 0 or width > MAX_PIXELS or height > MAX_PIXELS or width * height > MAX_PIXELS:
raise PermanentJobError("invalid_image")
try:
with Image.open(io.BytesIO(data)) as image:
image.load()
oriented = ImageOps.exif_transpose(image)
rgb = _rgb(oriented)
rgb.load()
return rgb
except DecompressionBombError as exc:
raise PermanentJobError("invalid_image") from exc
except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc:
raise PermanentJobError("invalid_image") from exc
def fit_inside(image, box):
width, height = image.size
if width <= box and height <= box:
return image.copy()
if width >= height:
new_w = box
new_h = max(1, (height * box) // width)
else:
new_h = box
new_w = max(1, (width * box) // height)
return image.resize((new_w, new_h), Image.Resampling.LANCZOS)
def center_square(image, size):
width, height = image.size
side = min(width, height)
left = (width - side) // 2
top = (height - side) // 2
cropped = image.crop((left, top, left + side, top + side))
if cropped.size != (size, size):
cropped = cropped.resize((size, size), Image.Resampling.LANCZOS)
return cropped
def _jpeg(image):
buffer = io.BytesIO()
image.save(buffer, format="JPEG", quality=JPEG_QUALITY, optimize=True)
return {
"bytes": buffer.getvalue(),
"content_type": "image/jpeg",
"width": image.size[0],
"height": image.size[1],
}
def _webp(image):
buffer = io.BytesIO()
image.save(buffer, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD)
return {
"bytes": buffer.getvalue(),
"content_type": "image/webp",
"width": image.size[0],
"height": image.size[1],
}
def render(data, declared):
rgb = load_rgb(data, declared)
size = header_size(data, declared)
thumb = fit_inside(rgb, THUMB_BOX)
crop = center_square(rgb, CROP_SIZE)
optimized = fit_inside(rgb, OPT_BOX)
return {
"original": {
"bytes": data,
"content_type": declared,
"width": size[0],
"height": size[1],
},
"thumbnail": _jpeg(thumb),
"crop": _jpeg(crop),
"optimized": _webp(optimized),
}
+896
View File
@@ -0,0 +1,896 @@
"""Состояния задания. Эффект один на (attachment_id, pipeline_version, вариант)."""
import hashlib
import json
import os
import random
import threading
import uuid
from datetime import datetime, timezone
from psycopg.pq import TransactionStatus
from psycopg.types.json import Json
from opentelemetry.trace import SpanKind, Status, StatusCode
from blobs import delete_quiet, put_bytes, read_bytes, variant_object_key
from errors import DependencyDown
from broker import publish_delay, publish_failed, publish_raw, broker_channel, PROCESS_QUEUE
from faults import take_fault
from images import PermanentJobError, TransientJobError, render
from telemetry import (
extract_context,
now_monotonic,
record_job,
record_time_to_ready,
start_span,
structured_log,
)
REQUIRED = ("original", "thumbnail", "crop", "optimized")
PUBLIC_CODES = ("invalid_image", "processing_failed", "retry_exhausted", "processing_timeout")
class LostLease(Exception):
pass
def _idle(conn):
if conn.info.transaction_status != TransactionStatus.IDLE:
conn.rollback()
def max_attempts():
raw = os.environ.get("JOB_MAX_ATTEMPTS", "5")
value = int(raw)
if not 1 <= value <= 20:
raise RuntimeError("JOB_MAX_ATTEMPTS")
return value
def lease_seconds():
raw = os.environ.get("JOB_TIMEOUT_SECONDS", "60")
value = int(raw)
if value < 5:
raise RuntimeError("JOB_TIMEOUT_SECONDS")
return value
def retention_seconds():
raw = os.environ.get("STAGING_RETENTION_SECONDS", "86400")
value = int(raw)
if value < 60:
raise RuntimeError("STAGING_RETENTION_SECONDS")
return value
def utc_now():
return datetime.now(timezone.utc)
def dump_time(value):
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
text = value.astimezone(timezone.utc).isoformat(timespec="microseconds")
if text.endswith("+00:00"):
text = text[:-6] + "Z"
return text
def backoff_seconds(failed_attempt):
base = min(30.0, 0.5 * (2 ** (failed_attempt - 1)))
return base + random.uniform(0, base * 0.25)
def job_message(job_id, attachment_id, pipeline_version, attempt, created_at, carrier=None):
if not isinstance(created_at, str):
created_at = dump_time(created_at)
payload = {
"schema_version": 1,
"job_id": str(job_id),
"type": "attachment.process.v1",
"attachment_id": str(attachment_id),
"pipeline_version": int(pipeline_version),
"attempt": int(attempt),
"created_at": created_at,
}
if carrier:
traceparent = carrier.get("traceparent")
tracestate = carrier.get("tracestate")
if isinstance(traceparent, str) and len(traceparent) == 55:
payload["traceparent"] = traceparent
if isinstance(tracestate, str) and 0 < len(tracestate) <= 512:
payload["tracestate"] = tracestate
return payload
def _uuid(value):
try:
return str(uuid.UUID(str(value)))
except (ValueError, AttributeError, TypeError):
return None
def _stamp(value):
if not isinstance(value, str) or not value.endswith("Z"):
return None
try:
datetime.fromisoformat(value[:-1] + "+00:00")
except ValueError:
return None
return value
def parse_job(body):
try:
payload = json.loads(body.decode("utf-8"))
except (UnicodeError, ValueError, AttributeError):
return None
if not isinstance(payload, dict):
return None
allowed = {
"schema_version",
"job_id",
"type",
"attachment_id",
"pipeline_version",
"attempt",
"created_at",
"traceparent",
"tracestate",
}
if set(payload) - allowed:
return None
required = allowed - {"traceparent", "tracestate"}
if not required <= set(payload):
return None
if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1":
return None
job_id = _uuid(payload.get("job_id"))
attachment_id = _uuid(payload.get("attachment_id"))
created_at = _stamp(payload.get("created_at"))
version = payload.get("pipeline_version")
attempt = payload.get("attempt")
if job_id is None or attachment_id is None or created_at is None:
return None
if type(version) is not int or type(attempt) is not int:
return None
if version < 1 or attempt < 1:
return None
if "traceparent" in payload:
traceparent = payload["traceparent"]
if not isinstance(traceparent, str) or len(traceparent) != 55:
return None
if "tracestate" in payload:
tracestate = payload["tracestate"]
if not isinstance(tracestate, str) or len(tracestate) > 512:
return None
payload["job_id"] = job_id
payload["attachment_id"] = attachment_id
return payload
def _event(conn, job, kind, detail=None):
conn.execute(
"""
INSERT INTO job_events (job_id, attachment_id, pipeline_version, kind, detail)
VALUES (%s, %s, %s, %s, %s)
""",
(job["id"], job["attachment_id"], job["pipeline_version"], kind, detail),
)
def _message_body(job, attempt, created_at):
payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], attempt, created_at)
return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
def fail_job(conn, job, code):
if code not in PUBLIC_CODES and code != "superseded":
code = "processing_failed"
_idle(conn)
with conn.transaction():
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'failed', error_code = %s, lease_owner = NULL,
lease_until = NULL, next_attempt_at = NULL, updated_at = now()
WHERE id = %s AND status IN ('queued', 'processing')
""",
(code, job["id"]),
)
if code in PUBLIC_CODES:
conn.execute(
"""
UPDATE attachments
SET status = 'failed', error_code = %s, updated_at = now()
WHERE id = %s AND pipeline_version = %s
AND status IN ('queued', 'processing')
""",
(code, job["attachment_id"], job["pipeline_version"]),
)
_event(conn, job, "failed", code)
try:
publish_failed(_message_body(job, job["attempt"], job["created_at"]))
except Exception:
# Реестр failed уже в базе. Повторная доставка увидит failed и подтвердит сообщение.
return
def schedule_retry(conn, job, code, owner):
attempt = int(job["attempt"])
if attempt >= int(job["max_attempts"]):
fail_job(conn, job, "retry_exhausted")
return
delay = backoff_seconds(attempt)
new_attempt = attempt + 1
created = utc_now()
payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], new_attempt, created)
_idle(conn)
with conn.transaction():
updated = conn.execute(
"""
UPDATE attachment_jobs
SET status = 'queued', attempt = %s, lease_owner = NULL, lease_until = NULL,
next_attempt_at = now() + make_interval(secs => %s),
error_code = NULL, updated_at = now()
WHERE id = %s AND status = 'processing' AND lease_owner = %s
RETURNING id
""",
(new_attempt, delay, job["id"], owner),
).fetchone()
if updated is None:
return
conn.execute(
"""
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = now()
WHERE id = %s AND pipeline_version = %s AND status = 'processing'
""",
(job["attachment_id"], job["pipeline_version"]),
)
conn.execute(
"""
INSERT INTO outbox (id, job_id, attempt, payload, created_at)
VALUES (%s, %s, %s, %s, %s)
""",
(uuid.uuid4(), job["id"], new_attempt, Json(payload), created),
)
_event(conn, job, "retry", code)
def release_expired(conn):
released = 0
_idle(conn)
with conn.transaction():
rows = conn.execute(
"""
UPDATE attachment_jobs
SET status = 'queued', lease_owner = NULL, lease_until = NULL, updated_at = now()
WHERE status = 'processing' AND lease_until < now()
RETURNING id, attachment_id, pipeline_version, attempt, created_at
"""
).fetchall()
for row in rows:
payload = job_message(row["id"], row["attachment_id"], row["pipeline_version"], row["attempt"], utc_now())
conn.execute(
"""
INSERT INTO outbox (id, job_id, attempt, payload, created_at)
SELECT %s, %s, %s, %s, now()
WHERE NOT EXISTS (
SELECT 1 FROM outbox WHERE job_id = %s AND published_at IS NULL
)
""",
(uuid.uuid4(), row["id"], row["attempt"], Json(payload), row["id"]),
)
conn.execute(
"""
UPDATE attachments
SET status = 'queued', updated_at = now()
WHERE id = %s AND pipeline_version = %s AND status = 'processing'
""",
(row["attachment_id"], row["pipeline_version"]),
)
_event(conn, row, "lease_expired", None)
released += 1
return released
def cleanup_staging(conn):
removed = 0
_idle(conn)
with conn.transaction():
rows = conn.execute(
"""
SELECT a.id, a.staging_key
FROM attachments a
WHERE a.staging_key IS NOT NULL
AND a.staging_deleted_at IS NULL
AND a.status IN ('ready', 'failed')
AND NOT EXISTS (
SELECT 1 FROM attachment_jobs j
WHERE j.attachment_id = a.id
AND j.status IN ('queued', 'processing')
)
"""
).fetchall()
for row in rows:
if delete_quiet(row["staging_key"]):
with conn.transaction():
conn.execute(
"""
UPDATE attachments
SET staging_deleted_at = now()
WHERE id = %s AND staging_deleted_at IS NULL
""",
(row["id"],),
)
removed += 1
return removed
def _load_job(conn, job_id):
return conn.execute(
"""
SELECT id, attachment_id, pipeline_version, status, attempt, max_attempts,
lease_owner, lease_until, next_attempt_at, error_code, created_at
FROM attachment_jobs
WHERE id = %s
""",
(job_id,),
).fetchone()
def _load_attachment(conn, attachment_id):
return conn.execute(
"""
SELECT id, content_type, sha256, object_key, staging_key, staging_deleted_at,
status, pipeline_version
FROM attachments
WHERE id = %s
""",
(attachment_id,),
).fetchone()
def _acquire(conn, job, owner, seconds):
_idle(conn)
with conn.transaction():
locked = conn.execute(
"""
UPDATE attachment_jobs
SET status = 'processing', lease_owner = %s,
lease_until = now() + make_interval(secs => %s),
updated_at = now()
WHERE id = %s
AND status IN ('queued', 'processing')
AND attempt = %s
AND pipeline_version = %s
AND (lease_owner IS NULL OR lease_owner = %s OR lease_until < now())
RETURNING id
""",
(owner, seconds, job["id"], job["attempt"], job["pipeline_version"], owner),
).fetchone()
if locked is None:
return False
conn.execute(
"""
UPDATE attachments
SET status = 'processing', updated_at = now()
WHERE id = %s AND pipeline_version = %s AND status IN ('queued', 'processing')
""",
(job["attachment_id"], job["pipeline_version"]),
)
_event(conn, job, "leased", None)
return True
def _variants(conn, attachment_id, version):
rows = conn.execute(
"""
SELECT name, object_key, sha256
FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = %s
""",
(attachment_id, version),
).fetchall()
return {row["name"]: row for row in rows}
def _source(conn, attachment):
if attachment["staging_key"] and attachment["staging_deleted_at"] is None:
data = read_bytes(attachment["staging_key"])
if data:
return data
data = read_bytes(attachment["object_key"])
if data:
return data
row = conn.execute(
"""
SELECT object_key
FROM attachment_variants
WHERE attachment_id = %s AND name = 'original'
ORDER BY pipeline_version DESC
LIMIT 1
""",
(attachment["id"],),
).fetchone()
if row is None:
raise TransientJobError("processing_failed")
data = read_bytes(row["object_key"])
if not data:
raise TransientJobError("processing_failed")
return data
def _render_limited(data, declared, seconds):
budget = max(5, seconds - 5)
box = []
errors = []
def run():
try:
box.append(render(data, declared))
except Exception as exc:
errors.append(exc)
thread = threading.Thread(target=run, daemon=True)
thread.start()
thread.join(budget)
if thread.is_alive():
raise TransientJobError("processing_timeout")
if errors:
exc = errors[0]
if isinstance(exc, (PermanentJobError, TransientJobError)):
raise exc
raise PermanentJobError("invalid_image")
return box[0]
def _beat(pool, job_id, owner, seconds, stop):
pause = max(2, seconds // 3)
while not stop.wait(pause):
try:
with pool.connection() as conn:
with conn.transaction():
conn.execute(
"""
UPDATE attachment_jobs
SET lease_until = now() + make_interval(secs => %s), updated_at = now()
WHERE id = %s AND lease_owner = %s AND status = 'processing'
""",
(seconds, job_id, owner),
)
except Exception:
return
def _supersede(conn, job):
_idle(conn)
with conn.transaction():
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'superseded', error_code = 'superseded', lease_owner = NULL,
lease_until = NULL, next_attempt_at = NULL, updated_at = now()
WHERE id = %s AND status IN ('queued', 'processing')
""",
(job["id"],),
)
_event(conn, job, "superseded", None)
def _finalize(conn, job, owner, rendered):
_idle(conn)
with conn.transaction():
current = conn.execute(
"""
SELECT lease_owner, status, pipeline_version
FROM attachment_jobs
WHERE id = %s
FOR UPDATE
""",
(job["id"],),
).fetchone()
if current is None or current["lease_owner"] != owner or current["status"] != "processing":
raise LostLease()
attachment = conn.execute(
"""
SELECT pipeline_version, status
FROM attachments
WHERE id = %s
FOR UPDATE
""",
(job["attachment_id"],),
).fetchone()
if attachment is None or int(attachment["pipeline_version"]) > int(job["pipeline_version"]):
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'superseded', error_code = 'superseded', lease_owner = NULL,
lease_until = NULL, updated_at = now()
WHERE id = %s
""",
(job["id"],),
)
_event(conn, job, "superseded", None)
return "superseded"
now = utc_now()
for name in REQUIRED:
item = rendered[name]
conn.execute(
"""
INSERT INTO attachment_variants (
attachment_id, pipeline_version, name, content_type, size_bytes,
width, height, sha256, object_key, created_at
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING
""",
(
job["attachment_id"],
job["pipeline_version"],
name,
item["content_type"],
len(item["bytes"]),
item["width"],
item["height"],
item["sha256"],
item["object_key"],
now,
),
)
stored = conn.execute(
"""
SELECT count(*) AS n
FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = %s
AND name IN ('original', 'thumbnail', 'crop', 'optimized')
""",
(job["attachment_id"], job["pipeline_version"]),
).fetchone()
if int(stored["n"]) < len(REQUIRED):
raise TransientJobError("processing_failed")
original = rendered["original"]
conn.execute(
"""
UPDATE attachments
SET status = 'ready', error_code = NULL, pipeline_version = %s,
object_key = %s, updated_at = %s
WHERE id = %s AND pipeline_version <= %s
""",
(
job["pipeline_version"],
original["object_key"],
now,
job["attachment_id"],
job["pipeline_version"],
),
)
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'ready', error_code = NULL, lease_owner = NULL,
lease_until = NULL, next_attempt_at = NULL, updated_at = %s
WHERE id = %s AND lease_owner = %s
""",
(now, job["id"], owner),
)
_event(conn, job, "ready", None)
return "ready"
def _drop_staging(conn, attachment):
_idle(conn)
with conn.transaction():
active = conn.execute(
"""
SELECT count(*) AS n
FROM attachment_jobs
WHERE attachment_id = %s AND status IN ('queued', 'processing')
""",
(attachment["id"],),
).fetchone()
if int(active["n"]) > 0 or not attachment["staging_key"]:
return
if delete_quiet(attachment["staging_key"]):
with conn.transaction():
conn.execute(
"""
UPDATE attachments
SET staging_deleted_at = now()
WHERE id = %s AND staging_deleted_at IS NULL
""",
(attachment["id"],),
)
def process_delivery(pool, body, owner):
payload = parse_job(body)
if payload is None:
return "ack"
carrier = {}
if "traceparent" in payload:
carrier["traceparent"] = payload["traceparent"]
if "tracestate" in payload:
carrier["tracestate"] = payload["tracestate"]
parent = extract_context(carrier) if carrier else None
started = now_monotonic()
with start_span(
"attachments.process consume",
kind=SpanKind.CONSUMER,
context=parent,
attributes={
"messaging.system": "rabbitmq",
"messaging.destination.name": PROCESS_QUEUE,
"messaging.operation": "process",
"job.type": "attachment.process.v1",
},
) as span:
return _process_delivery_body(pool, body, owner, payload, span, started)
def _process_delivery_body(pool, body, owner, payload, span, started):
import logging
logger = logging.getLogger("worker")
with pool.connection() as conn:
job = _load_job(conn, payload["job_id"])
if job is None:
return "ack"
if job["status"] in ("ready", "failed", "superseded"):
return "ack"
if int(payload["attempt"]) != int(job["attempt"]) or int(payload["pipeline_version"]) != int(job["pipeline_version"]):
return "ack"
if job["next_attempt_at"] is not None and job["next_attempt_at"] > utc_now():
delay = (job["next_attempt_at"] - utc_now()).total_seconds()
publish_delay(body, int(delay * 1000))
return "ack"
with pool.connection() as conn:
attachment = _load_attachment(conn, job["attachment_id"])
if attachment is None:
fail_job(conn, job, "invalid_image")
record_job("failed", now_monotonic() - started)
return "ack"
if int(attachment["pipeline_version"]) > int(job["pipeline_version"]):
_supersede(conn, job)
record_job("superseded", now_monotonic() - started)
return "ack"
if not _acquire(conn, job, owner, lease_seconds()):
publish_delay(body, 2000)
return "ack"
structured_log(logger, "job_leased", job_id=str(job["id"]), attempt=int(job["attempt"]))
stop = threading.Event()
beater = threading.Thread(
target=_beat,
args=(pool, job["id"], owner, lease_seconds(), stop),
daemon=True,
)
beater.start()
try:
with start_span("s3.read_source", kind=SpanKind.CLIENT):
with pool.connection() as conn:
if take_fault(conn, "crash_before_effect", job["attachment_id"]):
os_exit()
if take_fault(conn, "transient_once", job["attachment_id"]):
raise TransientJobError("processing_failed")
present = _variants(conn, job["attachment_id"], job["pipeline_version"])
source = None if set(REQUIRED) <= set(present) else _source(conn, attachment)
if source is None:
rendered = _rendered_from_rows(conn, job, present)
if source is not None:
if hashlib.sha256(source).hexdigest() != attachment["sha256"]:
raise PermanentJobError("invalid_image")
with start_span("image.render", kind=SpanKind.INTERNAL):
rendered = _render_limited(source, attachment["content_type"], lease_seconds())
with start_span("s3.write_variants", kind=SpanKind.CLIENT):
for name in REQUIRED:
item = rendered[name]
item["sha256"] = hashlib.sha256(item["bytes"]).hexdigest()
item["object_key"] = variant_object_key(job["attachment_id"], job["pipeline_version"], name)
if name in present:
continue
put_bytes(item["object_key"], item["bytes"], item["content_type"], utc_now())
with pool.connection() as conn:
if take_fault(conn, "crash_after_objects", job["attachment_id"]):
os_exit()
with start_span("db.finalize_ready", kind=SpanKind.CLIENT):
with pool.connection() as conn:
_finalize(conn, job, owner, rendered)
with pool.connection() as conn:
if take_fault(conn, "crash_after_commit", job["attachment_id"]):
os_exit()
with pool.connection() as conn:
_drop_staging(conn, attachment)
duration = now_monotonic() - started
record_job("ready", duration)
if job.get("created_at") is not None:
created = job["created_at"]
if created.tzinfo is None:
created = created.replace(tzinfo=timezone.utc)
record_time_to_ready((utc_now() - created).total_seconds())
structured_log(logger, "job_ready", job_id=str(job["id"]), duration_ms=round(duration * 1000, 3))
span.set_status(Status(StatusCode.OK))
return "ack"
except PermanentJobError as exc:
with pool.connection() as conn:
fail_job(conn, job, exc.code)
record_job("failed", now_monotonic() - started)
span.set_status(Status(StatusCode.ERROR, exc.code))
structured_log(logger, "job_failed", level=30, job_id=str(job["id"]), error_code=exc.code)
return "ack"
except LostLease:
return "ack"
except (TransientJobError, DependencyDown) as exc:
code = exc.code if isinstance(exc, TransientJobError) else "processing_failed"
with pool.connection() as conn:
schedule_retry(conn, job, code, owner)
record_job("retry", now_monotonic() - started)
span.set_status(Status(StatusCode.ERROR, code))
structured_log(logger, "job_retry", level=30, job_id=str(job["id"]), error_code=code)
return "ack"
finally:
stop.set()
def os_exit():
os._exit(75)
def _rendered_from_rows(conn, job, present):
# Повтор после уже записанного эффекта не перекодирует файлы.
rendered = {}
for name in REQUIRED:
row = present[name]
data = read_bytes(row["object_key"])
if data is None:
raise TransientJobError("processing_failed")
meta = conn.execute(
"""
SELECT content_type, width, height, sha256, object_key
FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = %s AND name = %s
""",
(job["attachment_id"], job["pipeline_version"], name),
).fetchone()
rendered[name] = {
"bytes": data,
"content_type": meta["content_type"],
"width": int(meta["width"]),
"height": int(meta["height"]),
"sha256": meta["sha256"],
"object_key": meta["object_key"],
}
return rendered
def bump_pipeline(conn, attachment_id):
_idle(conn)
with conn.transaction():
row = conn.execute(
"SELECT id, pipeline_version FROM attachments WHERE id = %s FOR UPDATE",
(attachment_id,),
).fetchone()
if row is None:
return None
version = int(row["pipeline_version"]) + 1
job_id = uuid.uuid4()
created = utc_now()
final_key = variant_object_key(attachment_id, version, "original")
conn.execute(
"""
UPDATE attachments
SET pipeline_version = %s, status = 'queued', error_code = NULL,
object_key = %s, updated_at = %s
WHERE id = %s
""",
(version, final_key, created, attachment_id),
)
attempts = max_attempts()
conn.execute(
"""
INSERT INTO attachment_jobs (
id, attachment_id, pipeline_version, status, attempt, max_attempts, created_at, updated_at
) VALUES (%s, %s, %s, 'queued', 1, %s, %s, %s)
""",
(job_id, attachment_id, version, attempts, created, created),
)
payload = job_message(job_id, attachment_id, version, 1, created)
conn.execute(
"""
INSERT INTO outbox (id, job_id, attempt, payload, created_at)
VALUES (%s, %s, 1, %s, %s)
""",
(uuid.uuid4(), job_id, Json(payload), created),
)
conn.execute(
"""
INSERT INTO job_events (job_id, attachment_id, pipeline_version, kind, detail)
VALUES (%s, %s, %s, 'version', %s)
""",
(job_id, attachment_id, version, str(version)),
)
return {"job_id": str(job_id), "pipeline_version": version}
def replay_job(conn, job_id):
_idle(conn)
with conn.transaction():
job = conn.execute(
"""
SELECT id, attachment_id, pipeline_version, status, attempt, max_attempts, created_at
FROM attachment_jobs
WHERE id = %s
FOR UPDATE
""",
(job_id,),
).fetchone()
if job is None:
return {"result": "missing"}
if job["status"] == "ready":
return {"result": "already_ready"}
names = conn.execute(
"""
SELECT name FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = %s
""",
(job["attachment_id"], job["pipeline_version"]),
).fetchall()
have = {row["name"] for row in names}
if set(REQUIRED) <= have:
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'ready', error_code = NULL, lease_owner = NULL,
lease_until = NULL, next_attempt_at = NULL, updated_at = now()
WHERE id = %s AND status <> 'ready'
""",
(job["id"],),
)
return {"result": "already_done"}
if job["status"] != "failed":
return {"result": "not_failed"}
created = utc_now()
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'queued', attempt = 1, error_code = NULL, lease_owner = NULL,
lease_until = NULL, next_attempt_at = NULL, updated_at = %s
WHERE id = %s
""",
(created, job["id"]),
)
conn.execute(
"""
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = %s
WHERE id = %s AND pipeline_version = %s AND status = 'failed'
""",
(created, job["attachment_id"], job["pipeline_version"]),
)
payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], 1, created)
conn.execute(
"""
INSERT INTO outbox (id, job_id, attempt, payload, created_at)
VALUES (%s, %s, 1, %s, %s)
""",
(uuid.uuid4(), job["id"], Json(payload), created),
)
_event(conn, job, "replay", None)
return {"result": "queued"}
def republish(conn, job_id, attempt=None):
job = conn.execute(
"""
SELECT id, attachment_id, pipeline_version, attempt, created_at
FROM attachment_jobs
WHERE id = %s
""",
(job_id,),
).fetchone()
if job is None:
return False
chosen = int(job["attempt"] if attempt is None else attempt)
body = _message_body(job, chosen, utc_now())
with broker_channel() as channel:
publish_raw(channel, PROCESS_QUEUE, body)
return True
+519
View File
@@ -0,0 +1,519 @@
import json
import logging
import os
import re
import time
import uuid
from contextlib import asynccontextmanager
import anyio
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response, StreamingResponse
from opentelemetry import trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from pydantic import ValidationError
from starlette.exceptions import HTTPException as StarletteHTTPException
from blobs import MAX_BYTES, declared_type, inspect_image, open_object
from errors import ApiError, AttachmentNotReady, AttachmentRejected, DependencyDown, IdempotencyConflict, error_body
from schemas import CreateChat, CreateMessage, CreateUser
from store import Store, instance_id
from telemetry import record_http, route_template, setup as setup_telemetry, structured_log
LOGGER = logging.getLogger("messenger")
store = Store()
INSTANCE_ID = instance_id()
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
MAX_JSON_BYTES = 65536
@asynccontextmanager
async def lifespan(_app):
setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-api"))
await anyio.to_thread.run_sync(store.startup)
yield
store.shutting_down = True
await anyio.to_thread.run_sync(store.shutdown)
app = FastAPI(
title="mtusi-messenger",
version="lab6",
lifespan=lifespan,
redirect_slashes=False,
docs_url="/docs",
redoc_url=None,
openapi_url="/openapi.json",
)
def request_id_of(request: Request):
rid = getattr(request.state, "request_id", None)
if rid:
return rid
return str(uuid.uuid4())
def stamp(request: Request, response):
response.headers["X-Request-Id"] = request_id_of(request)
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
def json_ok(request: Request, payload, status_code=200):
return stamp(request, JSONResponse(payload, status_code=status_code))
def json_error(request: Request, status, code, message, extra_headers=None):
headers = {"X-Request-Id": request_id_of(request), "X-Instance-Id": INSTANCE_ID}
if extra_headers:
headers.update(extra_headers)
return JSONResponse(error_body(code, message, request_id_of(request)), status_code=status, headers=headers)
@app.middleware("http")
async def wrap_request(request: Request, call_next):
incoming = request.headers.get("X-Request-Id", "")
if incoming and len(incoming) <= 200 and incoming.isprintable() and incoming.strip():
request.state.request_id = incoming.strip()
else:
request.state.request_id = str(uuid.uuid4())
if store.shutting_down and request.url.path != "/health/live":
return json_error(request, 503, "unavailable", "Сервис временно недоступен")
started = time.perf_counter()
route = route_template(request.url.path)
is_health = request.url.path in ("/health/live", "/health/ready")
if is_health:
response = await call_next(request)
return stamp(request, response)
tracer = trace.get_tracer("messenger")
with tracer.start_as_current_span(
f"{request.method} {route}",
kind=SpanKind.SERVER,
attributes={
"http.request.method": request.method,
"http.route": route,
"service.instance.id": INSTANCE_ID,
},
) as span:
span.set_attribute("http.request_id", request.state.request_id)
try:
response = await call_next(request)
except Exception as exc:
span.record_exception(exc)
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
raise
stamped = stamp(request, response)
span.set_attribute("http.response.status_code", stamped.status_code)
if stamped.status_code >= 500:
span.set_status(Status(StatusCode.ERROR))
duration = time.perf_counter() - started
record_http(request.method, request.url.path, stamped.status_code, duration)
structured_log(
LOGGER,
"http_request",
method=request.method,
route=route,
status=stamped.status_code,
request_id=request.state.request_id,
duration_ms=round(duration * 1000, 3),
)
return stamped
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
return json_error(request, exc.status, exc.code, exc.message, exc.headers)
@app.exception_handler(RequestValidationError)
async def request_validation_handler(request: Request, _exc: RequestValidationError):
return json_error(request, 400, "invalid_request", "Некорректный запрос")
@app.exception_handler(ValidationError)
async def pydantic_handler(request: Request, _exc: ValidationError):
return json_error(request, 400, "invalid_request", "Некорректный запрос")
@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
if exc.status_code == 404:
return json_error(request, 404, "not_found", "Объект не найден")
return json_error(request, exc.status_code if exc.status_code < 500 else 500, "invalid_request", "Некорректный запрос")
@app.exception_handler(Exception)
async def unexpected_handler(request: Request, exc: Exception):
if isinstance(exc, ApiError):
return json_error(request, exc.status, exc.code, exc.message, exc.headers)
LOGGER.error("unhandled %s", type(exc).__name__)
return json_error(request, 500, "internal_error", "Внутренняя ошибка")
def unavailable():
raise ApiError(503, "unavailable", "Сервис временно недоступен")
def bearer_unauthorized():
raise ApiError(401, "unauthorized", "Требуется авторизация", {"WWW-Authenticate": BEARER_CHALLENGE})
def basic_unauthorized():
raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": BASIC_CHALLENGE})
def invalid():
raise ApiError(400, "invalid_request", "Некорректный запрос")
async def call_store(func, *args):
try:
return await anyio.to_thread.run_sync(func, *args)
except DependencyDown:
unavailable()
except IdempotencyConflict:
raise ApiError(409, "conflict", "Ключ уже использован с другим телом")
except AttachmentRejected:
raise ApiError(409, "conflict", "Вложение нельзя прикрепить")
except AttachmentNotReady:
raise ApiError(409, "conflict", "Вложение недоступно")
def parse_basic(request: Request):
header = request.headers.get("Authorization", "")
if not header.lower().startswith("basic "):
basic_unauthorized()
raw = header.split(" ", 1)[1].strip()
try:
import base64
decoded = base64.b64decode(raw, validate=True).decode("utf-8")
except (ValueError, UnicodeError):
basic_unauthorized()
if ":" not in decoded:
basic_unauthorized()
username, password = decoded.split(":", 1)
return username, password
def parse_bearer(request: Request):
header = request.headers.get("Authorization", "")
# Basic на бизнес-методе не открывает сессию. X-User-Id не смотрим.
if not header.lower().startswith("bearer "):
bearer_unauthorized()
token = header.split(" ", 1)[1].strip()
if len(token) < 32 or len(token) > 512 or any(ch.isspace() for ch in token):
bearer_unauthorized()
return token
async def current_user(request: Request):
token = parse_bearer(request)
user = await call_store(store.user_for_token, token)
if user is None:
bearer_unauthorized()
request.state.user_id = user["id"]
request.state.token = token
return user
def parse_chat_id(chat_id):
try:
return str(uuid.UUID(chat_id))
except ValueError:
invalid()
def parse_limit(raw):
if raw is None:
return 50
if not isinstance(raw, str) or len(raw) > 3 or not raw.isdigit():
invalid()
value = int(raw)
if value < 1 or value > 100:
invalid()
return value
async def read_json(request: Request):
ctype = request.headers.get("content-type", "")
if "application/json" not in ctype.lower():
raise ApiError(415, "unsupported_media_type", "Нужен application/json")
length = request.headers.get("content-length")
if length is not None and (not length.isdigit() or int(length) > MAX_JSON_BYTES):
raise ApiError(413, "payload_too_large", "Слишком большое тело")
raw = await request.body()
if len(raw) > MAX_JSON_BYTES:
raise ApiError(413, "payload_too_large", "Слишком большое тело")
try:
payload = json.loads(raw)
except (ValueError, UnicodeError):
invalid()
if not isinstance(payload, dict):
invalid()
return payload
def parse_idempotency_key(request: Request):
key = request.headers.get("Idempotency-Key")
if key is None or not IDEMPOTENCY_RE.fullmatch(key):
invalid()
return key
@app.get("/health/live")
async def liveness(request: Request):
return json_ok(request, {"status": "ok"})
@app.get("/health/ready")
async def readiness(request: Request):
if store.shutting_down:
raise ApiError(503, "unavailable", "Сервис временно недоступен")
ok = await call_store(store.ready)
if not ok:
unavailable()
return json_ok(request, {"status": "ok"})
@app.post("/api/v1/users", status_code=201)
async def register_user(request: Request, body: CreateUser):
user = await call_store(store.create_user, body.username, body.display_name, body.password)
if user is None:
raise ApiError(409, "conflict", "Имя пользователя занято")
return json_ok(request, user, 201)
@app.post("/api/v1/auth/sessions", status_code=201)
async def create_session(request: Request):
username, password = parse_basic(request)
session = await call_store(store.login, username, password)
if session is None:
basic_unauthorized()
request.state.user_id = session["user"]["id"]
return json_ok(request, session, 201)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
async def revoke_session(request: Request):
user = await current_user(request)
request.state.user_id = user["id"]
removed = await call_store(store.revoke, request.state.token)
if not removed:
bearer_unauthorized()
return stamp(request, Response(status_code=204))
@app.get("/api/v1/users/me")
async def me(request: Request):
user = await current_user(request)
return json_ok(request, user)
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request):
user = await current_user(request)
payload = await read_json(request)
try:
body = CreateChat.model_validate(payload)
except ValidationError:
invalid()
try:
chat = await call_store(store.create_chat, body.title, body.member_ids, user["id"])
except ValueError:
invalid()
return json_ok(request, chat, 201)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
user = await current_user(request)
limit = parse_limit(request.query_params.get("limit"))
cursor = request.query_params.get("cursor")
if cursor is not None and (not cursor or len(cursor) > 512):
invalid()
try:
items, next_cursor = await call_store(store.list_chats, user["id"], limit, cursor)
except ValueError:
invalid()
return json_ok(request, {"items": items, "next_cursor": next_cursor})
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(request: Request, chat_id: str):
user = await current_user(request)
chat = await call_store(store.get_chat, parse_chat_id(chat_id), user["id"])
if chat is None:
raise ApiError(404, "not_found", "Объект не найден")
return json_ok(request, chat)
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(request: Request, chat_id: str):
user = await current_user(request)
canonical = parse_chat_id(chat_id)
payload = await read_json(request)
key = parse_idempotency_key(request)
try:
body = CreateMessage.model_validate(payload)
except ValidationError:
invalid()
try:
message = await call_store(
store.add_message, canonical, user["id"], body.text, key, body.attachment_ids
)
except ValueError:
invalid()
if message is None:
raise ApiError(404, "not_found", "Объект не найден")
return json_ok(request, message, 201)
@app.get("/api/v1/chats/{chat_id}/messages")
async def list_messages(request: Request, chat_id: str):
user = await current_user(request)
canonical = parse_chat_id(chat_id)
limit = parse_limit(request.query_params.get("limit"))
cursor = request.query_params.get("cursor")
if cursor is not None and (not cursor or len(cursor) > 512):
invalid()
try:
result = await call_store(store.list_messages, canonical, user["id"], limit, cursor)
except ValueError:
invalid()
if result is None:
raise ApiError(404, "not_found", "Объект не найден")
items, next_cursor = result
return json_ok(request, {"items": items, "next_cursor": next_cursor})
def check_filename(value):
if not isinstance(value, str) or not value or len(value) > 255:
invalid()
if "/" in value or "\\" in value or "\x00" in value:
invalid()
if any(ord(char) < 32 or ord(char) == 127 for char in value):
invalid()
return value
def parse_attachment_id(value):
try:
return str(uuid.UUID(value))
except (TypeError, ValueError):
invalid()
async def read_upload(request: Request):
header = request.headers.get("content-length")
if header is not None:
try:
announced = int(header)
except ValueError:
invalid()
if announced > MAX_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 МиБ")
if announced < 1:
raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG")
chunks = []
total = 0
async for chunk in request.stream():
total += len(chunk)
if total > MAX_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 МиБ")
chunks.append(chunk)
if total < 1:
raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG")
return b"".join(chunks)
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(request: Request, chat_id: str):
user = await current_user(request)
canonical = parse_chat_id(chat_id)
member = await call_store(store.is_member, canonical, user["id"])
if not member:
raise ApiError(404, "not_found", "Объект не найден")
filename = check_filename(request.query_params.get("filename"))
media = declared_type(request.headers.get("content-type"))
if media is None:
raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG")
data = await read_upload(request)
content_type = await anyio.to_thread.run_sync(inspect_image, data, media)
saved = await call_store(
store.save_attachment, canonical, user["id"], filename, content_type, data
)
if saved is None:
raise ApiError(404, "not_found", "Объект не найден")
return json_ok(request, saved, 202)
@app.get("/api/v1/attachments/{attachment_id}")
async def get_attachment(request: Request, attachment_id: str):
user = await current_user(request)
canonical = parse_attachment_id(attachment_id)
found = await call_store(store.get_attachment, canonical, user["id"])
if found is None:
raise ApiError(404, "not_found", "Объект не найден")
return json_ok(request, found)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def download_attachment(request: Request, attachment_id: str):
user = await current_user(request)
canonical = parse_attachment_id(attachment_id)
variant = request.query_params.get("variant")
if variant is None:
variant = "original"
if variant not in ("original", "thumbnail", "crop", "optimized"):
invalid()
opened = await call_store(store.open_attachment, canonical, user["id"], variant)
if opened is None:
raise ApiError(404, "not_found", "Объект не найден")
def chunks():
body = open_object(opened["object_key"])
if body is None:
return
try:
for piece in body.stream(64 * 1024):
yield piece
finally:
body.close()
body.release_conn()
probe = await anyio.to_thread.run_sync(open_object, opened["object_key"])
if probe is None:
raise ApiError(404, "not_found", "Объект не найден")
probe.close()
probe.release_conn()
response = StreamingResponse(chunks(), media_type=opened["content_type"])
response.headers["Content-Length"] = str(opened["size_bytes"])
return stamp(request, response)
def setup_logging():
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
if __name__ == "__main__":
import uvicorn
setup_logging()
setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-api"))
host = os.environ.get("APP_HOST", "0.0.0.0")
port = int(os.environ.get("APP_PORT", "8080"))
uvicorn.run(
app,
host=host,
port=port,
timeout_graceful_shutdown=5,
timeout_keep_alive=2,
access_log=False,
log_config=None,
)
+112
View File
@@ -0,0 +1,112 @@
"""Однократные миграции. Запускается отдельным контейнером, не каждой репликой API."""
import logging
import os
import sys
import time
from pathlib import Path
import psycopg
from psycopg.rows import dict_row
LOGGER = logging.getLogger("migrate")
MIGRATION_LOCK = 421003
CONNECT_TIMEOUT = 3
def read_secret_file(path):
with open(path, encoding="utf-8") as handle:
return handle.read().strip()
def database_url():
path = os.environ.get("DATABASE_URL_FILE")
if path:
return read_secret_file(path)
url = os.environ.get("DATABASE_URL", "").strip()
if not url:
raise RuntimeError("DATABASE_URL_FILE")
return url
def split_sql(script):
parts = []
buf = []
for line in script.splitlines():
if line.strip().startswith("--"):
continue
buf.append(line)
if line.strip().endswith(";"):
statement = "\n".join(buf).strip()
if statement.endswith(";"):
statement = statement[:-1].strip()
if statement:
parts.append(statement)
buf = []
tail = "\n".join(buf).strip().rstrip(";").strip()
if tail:
parts.append(tail)
return parts
def wait_postgres(conninfo):
last = None
for _ in range(60):
try:
with psycopg.connect(conninfo, connect_timeout=CONNECT_TIMEOUT) as conn:
conn.execute("SELECT 1")
return
except psycopg.OperationalError as exc:
last = type(exc).__name__
time.sleep(1)
LOGGER.error("postgres not ready: %s", last)
raise SystemExit(1)
def apply(conninfo):
directory = Path(__file__).resolve().parents[1] / "migrations"
with psycopg.connect(conninfo, connect_timeout=CONNECT_TIMEOUT, row_factory=dict_row) as conn:
conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK,))
conn.commit()
try:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
applied_at timestamptz NOT NULL DEFAULT now()
)
"""
)
conn.commit()
for path in sorted(directory.glob("*.sql")):
version = path.stem
found = conn.execute(
"SELECT 1 FROM schema_migrations WHERE version = %s",
(version,),
).fetchone()
conn.rollback()
if found:
LOGGER.info("migration %s already applied", version)
continue
with conn.transaction():
for statement in split_sql(path.read_text(encoding="utf-8")):
conn.execute(statement)
conn.execute(
"INSERT INTO schema_migrations (version) VALUES (%s)",
(version,),
)
LOGGER.info("migration %s applied", version)
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK,))
conn.commit()
def main():
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
conninfo = database_url()
wait_postgres(conninfo)
apply(conninfo)
return 0
if __name__ == "__main__":
sys.exit(main())
+26
View File
@@ -0,0 +1,26 @@
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHash, VerificationError, VerifyMismatchError
# Параметры ближе к учебному стенду, чем к максимальным: time=2, 19 MiB, Argon2id.
# Хеш всё равно медленный password KDF, не SHA-256 и не открытый текст.
hasher = PasswordHasher(
time_cost=2,
memory_cost=19456,
parallelism=1,
hash_len=32,
salt_len=16,
)
# Чтобы неизвестный username не отвечал заметно быстрее известного.
DUMMY_HASH = hasher.hash("dummy-password-not-a-user")
def hash_password(password):
return hasher.hash(password)
def verify_password(password, encoded):
try:
return hasher.verify(encoded, password)
except (VerifyMismatchError, VerificationError, InvalidHash):
return False
+174
View File
@@ -0,0 +1,174 @@
"""Сверка строк attachments и объектов bucket.
Свежий объект моложе grace не удаляется: загрузка могла записать файл
раньше, чем строка в базе. Строка, на которую ссылается сообщение, тоже
не удаляется, даже если файла уже нет. По умолчанию только план, без удалений.
"""
import argparse
import json
import os
import sys
import time
from datetime import datetime, timezone
import psycopg
from psycopg.rows import dict_row
from blobs import BUCKET, delete_quiet, list_objects
def grace_seconds():
raw = os.environ.get("RECONCILE_GRACE_SECONDS", "120")
value = int(raw)
if value < 1:
raise RuntimeError("RECONCILE_GRACE_SECONDS")
return value
def database_url():
path = os.environ.get("DATABASE_URL_FILE")
if not path:
raise RuntimeError("DATABASE_URL_FILE")
with open(path, encoding="utf-8") as handle:
return handle.read().strip()
def aware(value):
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def plan(grace):
started = time.perf_counter()
objects = list_objects()
listed = time.perf_counter()
now = datetime.now(timezone.utc)
with psycopg.connect(database_url(), row_factory=dict_row) as conn:
rows = conn.execute(
"""
SELECT a.id, a.object_key, a.staging_key, a.created_at, a.status,
EXISTS (
SELECT 1 FROM message_attachments m
WHERE m.attachment_id = a.id
) AS in_use
FROM attachments a
"""
).fetchall()
variant_rows = conn.execute("SELECT object_key FROM attachment_variants").fetchall()
by_key = {}
for row in rows:
by_key[row["object_key"]] = row
if row["staging_key"]:
by_key[row["staging_key"]] = row
for item in variant_rows:
by_key.setdefault(item["object_key"], {"status": "variant"})
object_keys = {item["key"] for item in objects}
actions = []
preserved = []
for item in objects:
row = by_key.get(item["key"])
if row is not None:
continue
created = aware(item["created_at"]) or now
age = (now - created).total_seconds()
entry = {"key": item["key"], "age_seconds": round(age, 3)}
if age < grace:
entry["reason"] = "fresh_object"
preserved.append(entry)
continue
entry["action"] = "delete_orphan_object"
actions.append(entry)
for row in rows:
if row["object_key"] in object_keys:
continue
created = aware(row["created_at"]) or now
age = (now - created).total_seconds()
entry = {
"attachment_id": str(row["id"]),
"key": row["object_key"],
"age_seconds": round(age, 3),
}
if row["status"] != "ready":
entry["reason"] = "not_ready"
preserved.append(entry)
continue
if row["in_use"]:
entry["reason"] = "missing_but_in_use"
preserved.append(entry)
continue
if age < grace:
entry["reason"] = "fresh_row"
preserved.append(entry)
continue
entry["action"] = "delete_orphan_row"
actions.append(entry)
return {
"bucket": BUCKET,
"grace_seconds": grace,
"objects": len(objects),
"rows": len(rows),
"list_seconds": round(listed - started, 6),
"plan_seconds": round(time.perf_counter() - started, 6),
"actions": actions,
"preserved": preserved,
}
def apply_plan(report):
deleted_objects = 0
deleted_rows = 0
with psycopg.connect(database_url(), row_factory=dict_row) as conn:
for action in report["actions"]:
if action["action"] == "delete_orphan_object":
if delete_quiet(action["key"]):
deleted_objects += 1
elif action["action"] == "delete_orphan_row":
with conn.transaction():
result = conn.execute(
"""
DELETE FROM attachments
WHERE id = %s
AND status = 'ready'
AND NOT EXISTS (
SELECT 1 FROM message_attachments m
WHERE m.attachment_id = attachments.id
)
AND NOT EXISTS (
SELECT 1 FROM attachment_jobs j
WHERE j.attachment_id = attachments.id
)
AND NOT EXISTS (
SELECT 1 FROM attachment_variants v
WHERE v.attachment_id = attachments.id
)
""",
(action["attachment_id"],),
)
deleted_rows += result.rowcount
report["deleted_objects"] = deleted_objects
report["deleted_rows"] = deleted_rows
return report
def main():
parser = argparse.ArgumentParser(description="Сверить вложения в базе и в bucket")
parser.add_argument("--apply", action="store_true", help="выполнить удаления; без флага только план")
parser.add_argument("--grace", type=int, default=None)
args = parser.parse_args()
grace = args.grace if args.grace is not None else grace_seconds()
if grace < 1:
print("grace должен быть не меньше 1 секунды", file=sys.stderr)
return 2
report = plan(grace)
report["apply"] = bool(args.apply)
if args.apply:
apply_plan(report)
print(json.dumps(report, ensure_ascii=False, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+24
View File
@@ -0,0 +1,24 @@
"""Явный повтор failed-задания. Уже готовый эффект второй раз не исполняется."""
import json
import sys
import psycopg
from psycopg.rows import dict_row
from blobs import read_secret
from jobs import replay_job
def main():
if len(sys.argv) != 2:
print("usage: replay.py JOB_ID", file=sys.stderr)
return 2
url = read_secret(__import__("os").environ["DATABASE_URL_FILE"])
with psycopg.connect(url, row_factory=dict_row) as conn:
result = replay_job(conn, sys.argv[1])
print(json.dumps(result, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+128
View File
@@ -0,0 +1,128 @@
import re
import uuid
from pydantic import BaseModel, ConfigDict, field_validator, model_validator
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
def _visible(value, limit):
if not isinstance(value, str) or not value.strip() or len(value) > limit:
raise ValueError("text")
return value
class CreateUser(StrictModel):
username: str
display_name: str
password: str
@field_validator("username")
@classmethod
def username_ok(cls, value):
if not isinstance(value, str) or not USERNAME_RE.fullmatch(value):
raise ValueError("username")
return value
@field_validator("display_name")
@classmethod
def display_name_ok(cls, value):
return _visible(value, 100)
@field_validator("password")
@classmethod
def password_ok(cls, value):
if not isinstance(value, str) or not value.strip():
raise ValueError("password")
# Длина в code points, не в байтах: bcrypt тут не подходит из-за усечения UTF-8.
if len(value) < 12 or len(value) > 128:
raise ValueError("password")
return value
class CreateChat(StrictModel):
title: str
member_ids: list
@field_validator("title")
@classmethod
def title_ok(cls, value):
return _visible(value, 100)
@field_validator("member_ids")
@classmethod
def member_ids_ok(cls, value):
if not isinstance(value, list) or not (1 <= len(value) <= 100):
raise ValueError("member_ids")
# Канонический UUID (нижний регистр). Иначе один и тот же id
# в разном регистре выглядит как два участника и упирается в PK.
normalized = []
seen = set()
for item in value:
if not isinstance(item, str):
raise ValueError("member_ids")
try:
canonical = str(uuid.UUID(item))
except ValueError as exc:
raise ValueError("member_ids") from exc
if canonical in seen:
raise ValueError("member_ids")
seen.add(canonical)
normalized.append(canonical)
return normalized
class CreateMessage(StrictModel):
text: str
attachment_ids: list = []
@model_validator(mode="before")
@classmethod
def fill_attachments(cls, value):
# Нет поля и явный [] — одно и то же тело для Idempotency-Key.
if not isinstance(value, dict):
return value
data = dict(value)
data.setdefault("attachment_ids", [])
return data
@field_validator("text")
@classmethod
def text_ok(cls, value):
if not isinstance(value, str) or len(value) > 4000:
raise ValueError("text")
if value != "" and not value.strip():
raise ValueError("text")
return value
@field_validator("attachment_ids")
@classmethod
def attachments_ok(cls, value):
if not isinstance(value, list) or len(value) > 10:
raise ValueError("attachment_ids")
normalized = []
seen = set()
for item in value:
if not isinstance(item, str):
raise ValueError("attachment_ids")
try:
canonical = str(uuid.UUID(item))
except ValueError as exc:
raise ValueError("attachment_ids") from exc
if canonical in seen:
raise ValueError("attachment_ids")
seen.add(canonical)
normalized.append(canonical)
return normalized
@model_validator(mode="after")
def text_or_file(self):
if self.text == "" and not self.attachment_ids:
raise ValueError("text")
if self.text and not self.text.strip():
raise ValueError("text")
return self
+924
View File
@@ -0,0 +1,924 @@
import base64
import hashlib
import json
import logging
import os
import secrets
import socket
import time
import uuid
from datetime import datetime, timedelta, timezone
import psycopg
import redis
from psycopg.errors import CheckViolation, ForeignKeyViolation, UniqueViolation
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from errors import AttachmentNotReady, AttachmentRejected, DependencyDown, IdempotencyConflict
from passwords import DUMMY_HASH, hash_password, verify_password
LOGGER = logging.getLogger("messenger")
POOL_TIMEOUT_SECONDS = 3
STATEMENT_TIMEOUT_MS = 5000
LOCK_TIMEOUT_MS = 3000
REDIS_MAX_CONNECTIONS = 16
REDIS_SOCKET_TIMEOUT = 2
SCHEMA_VERSION = "004_jobs"
SQL_MESSAGES_FIRST = """
SELECT id, chat_id, sender_id, body AS text, created_at
FROM messages
WHERE chat_id = %s
ORDER BY created_at, id
LIMIT %s
"""
SQL_MESSAGES_AFTER = """
SELECT id, chat_id, sender_id, body AS text, created_at
FROM messages
WHERE chat_id = %s
AND (created_at, id) > (%s, %s)
ORDER BY created_at, id
LIMIT %s
"""
SQL_MESSAGE_BY_ID = """
SELECT id, chat_id, sender_id, body AS text, created_at
FROM messages
WHERE id = %s
"""
def instance_id():
# Docker без hostname: в контейнере это id контейнера, он не меняется,
# пока живёт этот контейнер. stop-one.sh находит контейнер по этому значению.
return socket.gethostname()
def pool_max():
raw = os.environ.get("POOL_MAX", "10")
try:
value = int(raw)
except ValueError as exc:
raise RuntimeError("POOL_MAX") from exc
if not 1 <= value <= 32:
raise RuntimeError("POOL_MAX")
return value
def utc_now():
return datetime.now(timezone.utc)
def dump_time(value):
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
text = value.astimezone(timezone.utc).isoformat(timespec="microseconds")
if text.endswith("+00:00"):
text = text[:-6] + "Z"
return text
def parse_time(value):
if not isinstance(value, str) or not value.endswith("Z"):
raise ValueError("time")
return datetime.fromisoformat(value[:-1] + "+00:00")
def encode_cursor(scope, created_at, item_id):
raw = "{}|{}|{}".format(scope, created_at, item_id).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def decode_cursor(cursor, expected_scope):
if not isinstance(cursor, str) or not cursor or len(cursor) > 512:
raise ValueError("cursor")
pad = "=" * (-len(cursor) % 4)
try:
decoded = base64.urlsafe_b64decode(cursor + pad).decode("utf-8")
except (ValueError, UnicodeError) as exc:
raise ValueError("cursor") from exc
parts = decoded.split("|")
if len(parts) != 3:
raise ValueError("cursor")
scope, created_at, item_id = parts
if scope != expected_scope:
raise ValueError("cursor")
parse_time(created_at)
try:
uuid.UUID(item_id)
except ValueError as exc:
raise ValueError("cursor") from exc
return created_at, item_id
def session_key(token):
digest = hashlib.sha256(token.encode("utf-8")).hexdigest()
return "sess:" + digest
def message_fingerprint(text, attachment_ids):
# Канонический JSON: порядок ключей не важен, пропущенный attachment_ids равен [].
canonical = json.dumps(
{"attachment_ids": list(attachment_ids), "text": text},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def advisory_key(user_id, path, idem_key):
raw = "\0".join((user_id, "POST", path, idem_key)).encode("utf-8")
digest = hashlib.sha256(raw).digest()
return int.from_bytes(digest[:8], "big", signed=True)
def chat_path(chat_id):
return "/api/v1/chats/" + chat_id + "/messages"
def public_user(row):
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": dump_time(row["created_at"]),
}
def public_message(row, attachment_ids=None):
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"attachment_ids": list(attachment_ids or []),
"created_at": dump_time(row["created_at"]),
}
def public_variant(row):
return {
"name": row["name"],
"content_type": row["content_type"],
"size_bytes": int(row["size_bytes"]),
"width": int(row["width"]),
"height": int(row["height"]),
"sha256": row["sha256"],
}
def public_attachment(row, variants=None):
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": int(row["size_bytes"]),
"sha256": row["sha256"],
"status": row["status"],
"variants": [public_variant(item) for item in (variants or [])],
"error_code": row["error_code"],
"created_at": dump_time(row["created_at"]),
"updated_at": dump_time(row["updated_at"]),
}
def public_chat(row, member_ids=None):
members = member_ids if member_ids is not None else row["member_ids"]
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": [str(item) for item in members],
"created_at": dump_time(row["created_at"]),
}
def read_secret_file(path):
with open(path, encoding="utf-8") as handle:
return handle.read().strip()
def _db_down(exc):
raise DependencyDown() from exc
class Store:
def __init__(self):
self.pool = None
self.redis = None
self.ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
self.idem_ttl = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400"))
self.pool_size = pool_max()
self.shutting_down = False
if self.ttl < 1 or self.idem_ttl < 1:
raise RuntimeError("TTL")
kind = os.environ.get("SESSION_STORE", "redis").strip().lower()
if kind != "redis":
raise RuntimeError("SESSION_STORE")
def database_url(self):
path = os.environ.get("DATABASE_URL_FILE")
if path:
return read_secret_file(path)
url = os.environ.get("DATABASE_URL", "").strip()
if not url:
raise RuntimeError("DATABASE_URL_FILE")
return url
def redis_url(self):
path = os.environ.get("CACHE_URL_FILE")
if path:
return read_secret_file(path)
url = os.environ.get("CACHE_URL", "").strip()
if not url:
raise RuntimeError("CACHE_URL_FILE")
return url
def startup(self):
conninfo = self.database_url()
self._wait_schema(conninfo)
self.pool = ConnectionPool(
conninfo,
min_size=1,
max_size=self.pool_size,
timeout=POOL_TIMEOUT_SECONDS,
max_waiting=64,
max_lifetime=300,
kwargs={
"connect_timeout": POOL_TIMEOUT_SECONDS,
"row_factory": dict_row,
"application_name": "mtusi-lab6-" + instance_id(),
"options": "-c statement_timeout={} -c lock_timeout={} -c idle_in_transaction_session_timeout=15000".format(
STATEMENT_TIMEOUT_MS,
LOCK_TIMEOUT_MS,
),
},
open=True,
)
pool = redis.ConnectionPool.from_url(
self.redis_url(),
max_connections=REDIS_MAX_CONNECTIONS,
socket_timeout=REDIS_SOCKET_TIMEOUT,
socket_connect_timeout=REDIS_SOCKET_TIMEOUT,
health_check_interval=30,
decode_responses=True,
)
self.redis = redis.Redis(connection_pool=pool)
self._wait_redis()
self._wait_bucket()
try:
from broker import broker_channel, publish_pending
with broker_channel():
pass
with self._connect() as conn:
publish_pending(conn)
except Exception as exc:
LOGGER.warning("outbox startup deferred %s", type(exc).__name__)
self._start_queue_metrics()
LOGGER.info("ready instance=%s ttl=%s pool_max=%s", instance_id(), self.ttl, self.pool_size)
def _start_queue_metrics(self):
# Метрики возраста очереди должны жить и при остановленном worker.
import threading
def loop():
from broker import PROCESS_QUEUE, RETRY_QUEUE, queue_depth
from telemetry import observe_process, set_queue_stats
while not self.shutting_down:
try:
age = 0.0
with self._connect() as conn:
row = conn.execute(
"""
SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) AS age
FROM attachment_jobs
WHERE status IN ('queued', 'processing')
"""
).fetchone()
if row is not None and row["age"] is not None:
age = float(row["age"])
try:
depth = queue_depth(PROCESS_QUEUE)
except Exception:
depth = 0
try:
retry_depth = queue_depth(RETRY_QUEUE)
except Exception:
retry_depth = 0
set_queue_stats(PROCESS_QUEUE, depth, age)
set_queue_stats(RETRY_QUEUE, retry_depth, 0.0)
self.refresh_pool_stats()
observe_process()
except Exception as exc:
LOGGER.warning("queue metrics %s", type(exc).__name__)
time.sleep(5)
threading.Thread(target=loop, name="queue-metrics", daemon=True).start()
def shutdown(self):
self.shutting_down = True
if self.pool is not None:
self.pool.close()
if self.redis is not None:
self.redis.close()
def ready(self):
if self.shutting_down or self.pool is None or self.redis is None:
return False
try:
with self.pool.connection(timeout=POOL_TIMEOUT_SECONDS) as conn:
row = conn.execute(
"SELECT 1 FROM schema_migrations WHERE version = %s",
(SCHEMA_VERSION,),
).fetchone()
if row is None:
return False
self.redis.ping()
from blobs import bucket_ready
if not bucket_ready():
return False
except Exception:
LOGGER.warning("ready check failed")
return False
return True
def _wait_redis(self):
last = None
for _ in range(60):
try:
self.redis.ping()
return
except redis.RedisError as exc:
last = type(exc).__name__
time.sleep(1)
LOGGER.error("redis not ready: %s", last)
raise SystemExit(1)
def _wait_schema(self, conninfo):
# Схему накатывает сервис migrate. Реплики только ждут её, не применяют сами.
last = None
for _ in range(90):
try:
with psycopg.connect(conninfo, connect_timeout=POOL_TIMEOUT_SECONDS, row_factory=dict_row) as conn:
row = conn.execute(
"SELECT 1 FROM schema_migrations WHERE version = %s",
(SCHEMA_VERSION,),
).fetchone()
if row is not None:
return
last = "migration missing"
except psycopg.Error as exc:
last = type(exc).__name__
time.sleep(1)
LOGGER.error("schema not ready: %s", last)
raise SystemExit(1)
def _wait_bucket(self):
from blobs import bucket_ready
for _ in range(60):
if bucket_ready():
return
time.sleep(1)
LOGGER.error("object storage not ready")
raise SystemExit(1)
def refresh_pool_stats(self):
if self.pool is None:
return
try:
from telemetry import set_pool_stats
stats = self.pool.get_stats()
used = int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0))
waiting = int(stats.get("requests_waiting", 0))
set_pool_stats(max(0, used), waiting)
except Exception:
pass
def _connect(self):
self.refresh_pool_stats()
try:
return self.pool.connection(timeout=POOL_TIMEOUT_SECONDS)
except PoolTimeout as exc:
_db_down(exc)
def create_user(self, username, display_name, password):
password_hash = hash_password(password)
user_id = uuid.uuid4()
created = utc_now()
try:
with self._connect() as conn:
with conn.transaction():
row = conn.execute(
"""
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (username) DO NOTHING
RETURNING id, username, display_name, created_at
""",
(user_id, username, display_name, password_hash, created),
).fetchone()
except psycopg.OperationalError as exc:
_db_down(exc)
if row is None:
return None
return public_user(row)
def login(self, username, password):
try:
with self._connect() as conn:
row = conn.execute(
"""
SELECT id, username, display_name, password_hash, created_at
FROM users
WHERE username = %s
""",
(username,),
).fetchone()
except psycopg.OperationalError as exc:
_db_down(exc)
if row is None:
verify_password(password, DUMMY_HASH)
return None
if not verify_password(password, row["password_hash"]):
return None
return self._issue(public_user(row))
def _issue(self, user):
token = secrets.token_urlsafe(32)
expires = utc_now() + timedelta(seconds=self.ttl)
payload = json.dumps(
{"user_id": user["id"], "expires_at": dump_time(expires)},
separators=(",", ":"),
)
# pxat — абсолютное время. Рестарт Redis не сдвигает срок.
# В значении тоже лежит expires_at: даже если TTL ключа разъедется, сессия кончится вовремя.
try:
self.redis.set(session_key(token), payload, pxat=int(expires.timestamp() * 1000))
except redis.RedisError as exc:
raise DependencyDown() from exc
return {
"token": token,
"token_type": "Bearer",
"expires_at": dump_time(expires),
"user": user,
}
def user_for_token(self, token):
key = session_key(token)
try:
raw = self.redis.get(key)
except redis.RedisError as exc:
raise DependencyDown() from exc
if not raw:
return None
try:
payload = json.loads(raw)
expires = parse_time(payload["expires_at"])
user_id = payload["user_id"]
uuid.UUID(user_id)
except (ValueError, KeyError, TypeError):
return None
if utc_now() >= expires:
try:
self.redis.delete(key)
except redis.RedisError as exc:
raise DependencyDown() from exc
return None
try:
with self._connect() as conn:
row = conn.execute(
"""
SELECT id, username, display_name, created_at
FROM users
WHERE id = %s
""",
(user_id,),
).fetchone()
except psycopg.OperationalError as exc:
_db_down(exc)
if row is None:
return None
return public_user(row)
def revoke(self, token):
try:
deleted = self.redis.delete(session_key(token))
except redis.RedisError as exc:
raise DependencyDown() from exc
return deleted == 1
def create_chat(self, title, member_ids, caller_id):
try:
caller_uuid = uuid.UUID(str(caller_id))
member_uuids = []
seen = set()
for item in member_ids:
member_uuid = uuid.UUID(str(item))
if member_uuid in seen:
raise ValueError("members")
seen.add(member_uuid)
member_uuids.append(member_uuid)
except ValueError as exc:
raise ValueError("members") from exc
if caller_uuid not in seen:
raise ValueError("members")
chat_id = uuid.uuid4()
created = utc_now()
canonical_members = [str(item) for item in member_uuids]
try:
with self._connect() as conn:
try:
with conn.transaction():
conn.execute(
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
(chat_id, title, created),
)
for ordinal, member_id in enumerate(member_uuids):
conn.execute(
"""
INSERT INTO chat_members (chat_id, user_id, ordinal)
VALUES (%s, %s, %s)
""",
(chat_id, member_id, ordinal),
)
row = conn.execute(
"SELECT id, title, created_at FROM chats WHERE id = %s",
(chat_id,),
).fetchone()
except ForeignKeyViolation as exc:
raise ValueError("members") from exc
except UniqueViolation as exc:
raise ValueError("members") from exc
except psycopg.OperationalError as exc:
_db_down(exc)
return public_chat(row, canonical_members)
def _member(self, conn, chat_id, user_id):
return conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone()
def get_chat(self, chat_id, user_id):
try:
with self._connect() as conn:
if self._member(conn, chat_id, user_id) is None:
return None
row = conn.execute(
"""
SELECT c.id, c.title, c.created_at,
array_agg(m.user_id ORDER BY m.ordinal) AS member_ids
FROM chats c
JOIN chat_members m ON m.chat_id = c.id
WHERE c.id = %s
GROUP BY c.id
""",
(chat_id,),
).fetchone()
except psycopg.OperationalError as exc:
_db_down(exc)
if row is None:
return None
return public_chat(row)
def list_chats(self, user_id, limit, cursor):
after = None
if cursor is not None:
created_at, item_id = decode_cursor(cursor, "chats:" + user_id)
after = (parse_time(created_at), uuid.UUID(item_id))
sql = """
SELECT c.id, c.title, c.created_at,
array_agg(m.user_id ORDER BY m.ordinal) AS member_ids
FROM chats c
JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = %s
JOIN chat_members m ON m.chat_id = c.id
"""
params = [user_id]
if after is not None:
sql += " WHERE (c.created_at, c.id) > (%s, %s)"
params.extend(after)
sql += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT %s"
params.append(limit + 1)
try:
with self._connect() as conn:
rows = conn.execute(sql, params).fetchall()
except psycopg.OperationalError as exc:
_db_down(exc)
page = [public_chat(row) for row in rows[:limit]]
if len(rows) > limit and page:
last = page[-1]
return page, encode_cursor("chats:" + user_id, last["created_at"], last["id"])
return page, None
def _attachment_ids(self, conn, message_ids):
if not message_ids:
return {}
rows = conn.execute(
"""
SELECT message_id, attachment_id
FROM message_attachments
WHERE message_id = ANY(%s::uuid[])
ORDER BY ordinal
""",
(list(message_ids),),
).fetchall()
found = {}
for row in rows:
found.setdefault(str(row["message_id"]), []).append(str(row["attachment_id"]))
return found
def is_member(self, chat_id, user_id):
try:
with self._connect() as conn:
return self._member(conn, chat_id, user_id) is not None
except psycopg.OperationalError as exc:
_db_down(exc)
def _variants(self, conn, attachment_id, pipeline_version):
return conn.execute(
"""
SELECT name, content_type, size_bytes, width, height, sha256
FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = %s AND name <> 'original'
ORDER BY CASE name
WHEN 'thumbnail' THEN 1
WHEN 'crop' THEN 2
WHEN 'optimized' THEN 3
ELSE 4
END
""",
(attachment_id, pipeline_version),
).fetchall()
def save_attachment(self, chat_id, owner_id, filename, content_type, data):
from psycopg.types.json import Json
from blobs import delete_quiet, put_bytes, staging_object_key, variant_object_key
from broker import publish_pending
from faults import take_fault
from jobs import job_message, max_attempts
from telemetry import inject_trace
attachment_id = uuid.uuid4()
job_id = uuid.uuid4()
created = utc_now()
digest = hashlib.sha256(data).hexdigest()
staging = staging_object_key(attachment_id)
final_key = variant_object_key(attachment_id, 1, "original")
carrier = inject_trace({})
payload = job_message(job_id, attachment_id, 1, 1, created, carrier=carrier)
attempts = max_attempts()
put_bytes(staging, data, content_type, created)
try:
with self._connect() as conn:
with conn.transaction():
if self._member(conn, chat_id, owner_id) is None:
raise LookupError("chat")
row = conn.execute(
"""
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type,
size_bytes, sha256, object_key, staging_key, status,
pipeline_version, error_code, created_at, updated_at
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'queued', 1, NULL, %s, %s)
RETURNING id, chat_id, owner_id, filename, content_type,
size_bytes, sha256, status, error_code, created_at, updated_at
""",
(
attachment_id,
chat_id,
owner_id,
filename,
content_type,
len(data),
digest,
final_key,
staging,
created,
created,
),
).fetchone()
conn.execute(
"""
INSERT INTO attachment_jobs (
id, attachment_id, pipeline_version, status, attempt,
max_attempts, created_at, updated_at
)
VALUES (%s, %s, 1, 'queued', 1, %s, %s, %s)
""",
(job_id, attachment_id, attempts, created, created),
)
conn.execute(
"""
INSERT INTO outbox (id, job_id, attempt, payload, created_at)
VALUES (%s, %s, 1, %s, %s)
""",
(uuid.uuid4(), job_id, Json(payload), created),
)
except (LookupError, ForeignKeyViolation):
delete_quiet(staging)
return None
except psycopg.OperationalError as exc:
delete_quiet(staging)
_db_down(exc)
except Exception:
delete_quiet(staging)
raise
try:
with self._connect() as conn:
skipped = take_fault(conn, "skip_inline_publish", attachment_id)
if not skipped:
with self._connect() as conn:
publish_pending(conn, only_job_id=job_id)
except Exception as exc:
LOGGER.warning("publish deferred %s", type(exc).__name__)
return public_attachment(row, [])
def get_attachment(self, attachment_id, user_id):
try:
with self._connect() as conn:
row = conn.execute(
"""
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
a.size_bytes, a.sha256, a.status, a.error_code, a.pipeline_version,
a.created_at, a.updated_at
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s
WHERE a.id = %s
""",
(user_id, attachment_id),
).fetchone()
variants = []
if row is not None and row["status"] == "ready":
variants = self._variants(conn, row["id"], row["pipeline_version"])
except psycopg.OperationalError as exc:
_db_down(exc)
if row is None:
return None
return public_attachment(row, variants)
def open_attachment(self, attachment_id, user_id, variant):
try:
with self._connect() as conn:
row = conn.execute(
"""
SELECT a.object_key, a.content_type, a.size_bytes, a.status, a.pipeline_version
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s
WHERE a.id = %s
""",
(user_id, attachment_id),
).fetchone()
if row is None:
return None
if row["status"] != "ready":
raise AttachmentNotReady()
if variant == "original":
return {
"object_key": row["object_key"],
"content_type": row["content_type"],
"size_bytes": int(row["size_bytes"]),
}
found = conn.execute(
"""
SELECT object_key, content_type, size_bytes
FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = %s AND name = %s
""",
(attachment_id, row["pipeline_version"], variant),
).fetchone()
except psycopg.OperationalError as exc:
_db_down(exc)
if found is None:
return None
return {
"object_key": found["object_key"],
"content_type": found["content_type"],
"size_bytes": int(found["size_bytes"]),
}
def add_message(self, chat_id, sender_id, text, idem_key, attachment_ids):
# Один ключ сериализуется между репликами через advisory lock в той же
# транзакции, где INSERT сообщения. Откат не оставляет ключ занятым.
path = chat_path(chat_id)
digest = message_fingerprint(text, attachment_ids)
lock_key = advisory_key(sender_id, path, idem_key)
message_id = uuid.uuid4()
try:
with self._connect() as conn:
with conn.transaction():
# Иначе lock_timeout сессии (3 с) оборвёт очередь одинаковых ключей.
conn.execute("SET LOCAL lock_timeout = '15s'")
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
if self._member(conn, chat_id, sender_id) is None:
return None
existing = conn.execute(
"""
SELECT request_hash, message_id,
created_at >= now() - make_interval(secs => %s) AS fresh
FROM idempotency_keys
WHERE user_id = %s AND http_method = 'POST'
AND request_path = %s AND idem_key = %s
""",
(self.idem_ttl, sender_id, path, idem_key),
).fetchone()
if existing is not None and existing["fresh"]:
if existing["request_hash"] != digest:
raise IdempotencyConflict()
row = conn.execute(SQL_MESSAGE_BY_ID, (existing["message_id"],)).fetchone()
if row is None:
raise RuntimeError("idempotency target missing")
linked = self._attachment_ids(conn, [row["id"]]).get(str(row["id"]), [])
return public_message(row, linked)
if existing is not None:
conn.execute(
"""
DELETE FROM idempotency_keys
WHERE user_id = %s AND http_method = 'POST'
AND request_path = %s AND idem_key = %s
""",
(sender_id, path, idem_key),
)
if attachment_ids:
found = conn.execute(
"""
SELECT id, chat_id, status
FROM attachments
WHERE id = ANY(%s::uuid[])
""",
(list(attachment_ids),),
).fetchall()
by_id = {str(item["id"]): item for item in found}
for attachment_id in attachment_ids:
item = by_id.get(attachment_id)
if item is None or str(item["chat_id"]) != chat_id:
return None
if item["status"] == "failed" or item["status"] not in ("queued", "processing", "ready"):
raise AttachmentRejected()
row = conn.execute(
"""
INSERT INTO messages (id, chat_id, sender_id, body, created_at)
VALUES (%s, %s, %s, %s, clock_timestamp())
RETURNING id, chat_id, sender_id, body AS text, created_at
""",
(message_id, chat_id, sender_id, text),
).fetchone()
for ordinal, attachment_id in enumerate(attachment_ids):
conn.execute(
"""
INSERT INTO message_attachments (message_id, attachment_id, ordinal)
VALUES (%s, %s, %s)
""",
(row["id"], attachment_id, ordinal),
)
conn.execute(
"""
INSERT INTO idempotency_keys (
user_id, http_method, request_path, idem_key,
request_hash, message_id, created_at
)
VALUES (%s, 'POST', %s, %s, %s, %s, clock_timestamp())
""",
(sender_id, path, idem_key, digest, row["id"]),
)
return public_message(row, attachment_ids)
except CheckViolation as exc:
raise ValueError("message") from exc
except psycopg.OperationalError as exc:
_db_down(exc)
def list_messages(self, chat_id, user_id, limit, cursor):
after = None
if cursor is not None:
created_at, item_id = decode_cursor(cursor, chat_id)
after = (parse_time(created_at), uuid.UUID(item_id))
try:
with self._connect() as conn:
if self._member(conn, chat_id, user_id) is None:
return None
if after is None:
rows = conn.execute(SQL_MESSAGES_FIRST, (chat_id, limit + 1)).fetchall()
else:
rows = conn.execute(
SQL_MESSAGES_AFTER,
(chat_id, after[0], after[1], limit + 1),
).fetchall()
linked = self._attachment_ids(conn, [row["id"] for row in rows[:limit]])
except psycopg.OperationalError as exc:
_db_down(exc)
page = [public_message(row, linked.get(str(row["id"]), [])) for row in rows[:limit]]
if len(rows) > limit and page:
last = page[-1]
return page, encode_cursor(chat_id, last["created_at"], last["id"])
return page, None
+448
View File
@@ -0,0 +1,448 @@
"""OpenTelemetry: traces, metrics, structured logs. Секреты и тела сообщений не экспортируются."""
from __future__ import annotations
import json
import logging
import os
import socket
import time
from contextlib import contextmanager
from opentelemetry import metrics, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.metrics import CallbackOptions, Observation
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ALWAYS_ON, ParentBased, TraceIdRatioBased
from opentelemetry.trace import Link, SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
SERVICE_VERSION = "lab6"
PROPAGATOR = TraceContextTextMapPropagator()
_CONFIGURED = False
_TRACER = None
_METER = None
_STATE = {
"queue_depth": {},
"oldest_job_age": {},
"db_pool_used": 0.0,
"db_pool_waiting": 0.0,
"worker_busy": 0.0,
"worker_capacity": 0.0,
"process_cpu": 0.0,
"process_rss": 0.0,
}
http_requests = None
http_duration = None
jobs_processed = None
job_duration = None
e2e_ready = None
def _endpoint():
return os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
def _headers():
raw = os.environ.get("OTEL_EXPORTER_OTLP_HEADERS", "")
if not raw:
return {}
out = {}
for part in raw.split(","):
if "=" not in part:
continue
key, value = part.split("=", 1)
out[key.strip()] = value.strip()
return out
def _sampler():
ratio = os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0")
try:
value = float(ratio)
except ValueError:
value = 1.0
if value >= 1.0:
return ParentBased(ALWAYS_ON)
if value <= 0.0:
return ParentBased(TraceIdRatioBased(0.0))
return ParentBased(TraceIdRatioBased(value))
def _resource(service_name):
attrs = {
"service.name": service_name,
"service.version": os.environ.get("SERVICE_VERSION", SERVICE_VERSION),
"service.instance.id": socket.gethostname(),
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
}
extra = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
for part in extra.split(","):
if "=" not in part:
continue
key, value = part.split("=", 1)
key = key.strip()
value = value.strip()
if key and value and key not in attrs:
attrs[key] = value
return Resource.create(attrs)
def _obs_queue_depth(_options: CallbackOptions):
for name, value in list(_STATE["queue_depth"].items()):
yield Observation(float(value), {"messaging.destination.name": name})
def _obs_oldest(_options: CallbackOptions):
for name, value in list(_STATE["oldest_job_age"].items()):
yield Observation(float(value), {"messaging.destination.name": name})
def _obs_db_used(_options: CallbackOptions):
yield Observation(float(_STATE["db_pool_used"]))
def _obs_db_waiting(_options: CallbackOptions):
yield Observation(float(_STATE["db_pool_waiting"]))
def _obs_worker_busy(_options: CallbackOptions):
yield Observation(float(_STATE["worker_busy"]))
def _obs_worker_capacity(_options: CallbackOptions):
yield Observation(float(_STATE["worker_capacity"]))
def _obs_cpu(_options: CallbackOptions):
yield Observation(float(_STATE["process_cpu"]))
def _obs_rss(_options: CallbackOptions):
yield Observation(float(_STATE["process_rss"]))
def attach_log_handler(logger_provider):
handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
root = logging.getLogger()
root.setLevel(logging.INFO)
if not any(isinstance(item, LoggingHandler) for item in root.handlers):
root.addHandler(handler)
for name in ("messenger", "worker", "uvicorn", "uvicorn.error", "uvicorn.access"):
logging.getLogger(name).setLevel(logging.INFO)
def setup(service_name):
global _CONFIGURED, _TRACER, _METER
global http_requests, http_duration, jobs_processed, job_duration, e2e_ready
if _CONFIGURED:
# uvicorn мог переустановить handlers после импорта модуля
provider = getattr(setup, "_logger_provider", None)
if provider is not None:
attach_log_handler(provider)
return
if os.environ.get("OTEL_SDK_DISABLED", "").lower() in ("1", "true", "yes"):
_CONFIGURED = True
return
resource = _resource(service_name)
base = _endpoint()
hdr = _headers()
provider = TracerProvider(resource=resource, sampler=_sampler())
provider.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(endpoint=f"{base}/v1/traces", headers=hdr, timeout=5),
max_queue_size=512,
max_export_batch_size=128,
schedule_delay_millis=2000,
export_timeout_millis=5000,
)
)
trace.set_tracer_provider(provider)
_TRACER = trace.get_tracer("messenger", SERVICE_VERSION)
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=f"{base}/v1/metrics", headers=hdr, timeout=5),
export_interval_millis=10000,
export_timeout_millis=5000,
)
views = (
View(
instrument_name="messenger.http.duration",
aggregation=ExplicitBucketHistogramAggregation(
[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]
),
),
View(
instrument_name="messenger.jobs.duration",
aggregation=ExplicitBucketHistogramAggregation(
[0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 15.0, 30.0, 60.0]
),
),
View(
instrument_name="messenger.attachment.time_to_ready",
aggregation=ExplicitBucketHistogramAggregation(
[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0]
),
),
)
meter_provider = MeterProvider(resource=resource, metric_readers=[reader], views=views)
metrics.set_meter_provider(meter_provider)
_METER = metrics.get_meter("messenger", SERVICE_VERSION)
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(endpoint=f"{base}/v1/logs", headers=hdr, timeout=5),
max_queue_size=512,
max_export_batch_size=128,
schedule_delay_millis=2000,
export_timeout_millis=5000,
)
)
set_logger_provider(logger_provider)
setup._logger_provider = logger_provider
attach_log_handler(logger_provider)
http_requests = _METER.create_counter(
"messenger.http.requests",
unit="1",
description="HTTP requests by method, route template and status class",
)
http_duration = _METER.create_histogram(
"messenger.http.duration",
unit="s",
description="HTTP request duration in seconds",
)
jobs_processed = _METER.create_counter(
"messenger.jobs.processed",
unit="1",
description="Jobs finished by type and outcome",
)
job_duration = _METER.create_histogram(
"messenger.jobs.duration",
unit="s",
description="Job processing duration in seconds",
)
e2e_ready = _METER.create_histogram(
"messenger.attachment.time_to_ready",
unit="s",
description="Seconds from durable acceptance to ready",
)
_METER.create_observable_up_down_counter(
"messenger.queue.depth",
callbacks=[_obs_queue_depth],
unit="1",
description="Approximate broker queue depth",
)
_METER.create_observable_gauge(
"messenger.queue.oldest_job_age",
callbacks=[_obs_oldest],
unit="s",
description="Age of the oldest unfinished job",
)
_METER.create_observable_gauge(
"messenger.db.pool.used",
callbacks=[_obs_db_used],
unit="1",
description="Checked-out DB pool connections",
)
_METER.create_observable_gauge(
"messenger.db.pool.waiting",
callbacks=[_obs_db_waiting],
unit="1",
description="Waiters for a DB pool connection",
)
_METER.create_observable_gauge(
"messenger.worker.busy",
callbacks=[_obs_worker_busy],
unit="1",
description="Busy worker slots",
)
_METER.create_observable_gauge(
"messenger.worker.capacity",
callbacks=[_obs_worker_capacity],
unit="1",
description="Configured worker concurrency",
)
_METER.create_observable_gauge(
"messenger.process.cpu_seconds",
callbacks=[_obs_cpu],
unit="s",
description="Process user+system CPU seconds",
)
_METER.create_observable_gauge(
"messenger.process.rss_bytes",
callbacks=[_obs_rss],
unit="By",
description="Process RSS bytes",
)
_CONFIGURED = True
def tracer():
return _TRACER or trace.get_tracer("messenger", SERVICE_VERSION)
def status_class(code):
return f"{int(code) // 100}xx"
def route_template(path):
if path in ("/health/live", "/health/ready"):
return path
parts = [p for p in path.strip("/").split("/") if p]
if len(parts) >= 3 and parts[0] == "api" and parts[1] == "v1":
if parts[2] == "users" and len(parts) == 3:
return "/api/v1/users"
if parts[2] == "users" and len(parts) == 4 and parts[3] == "me":
return "/api/v1/users/me"
if parts[2] == "auth" and len(parts) >= 4 and parts[3] == "sessions":
if len(parts) == 4:
return "/api/v1/auth/sessions"
return "/api/v1/auth/sessions/current"
if parts[2] == "chats" and len(parts) == 3:
return "/api/v1/chats"
if parts[2] == "chats" and len(parts) == 4:
return "/api/v1/chats/{chat_id}"
if parts[2] == "chats" and len(parts) == 5 and parts[4] == "messages":
return "/api/v1/chats/{chat_id}/messages"
if parts[2] == "chats" and len(parts) == 5 and parts[4] == "attachments":
return "/api/v1/chats/{chat_id}/attachments"
if parts[2] == "attachments" and len(parts) == 4:
return "/api/v1/attachments/{attachment_id}"
if parts[2] == "attachments" and len(parts) == 5 and parts[4] == "content":
return "/api/v1/attachments/{attachment_id}/content"
return "other"
def record_http(method, path, status, duration_s):
if http_requests is None:
return
if path in ("/health/live", "/health/ready"):
return
route = route_template(path)
attrs = {
"http.request.method": method,
"http.route": route,
"http.response.status_class": status_class(status),
}
http_requests.add(1, attrs)
http_duration.record(duration_s, attrs)
def inject_trace(carrier):
PROPAGATOR.inject(carrier)
return carrier
def extract_context(carrier):
return PROPAGATOR.extract(carrier)
def current_trace_fields():
span = trace.get_current_span()
ctx = span.get_span_context()
if not ctx.is_valid:
return {}, None, None
carrier = {}
inject_trace(carrier)
return carrier, format(ctx.trace_id, "032x"), format(ctx.span_id, "016x")
def structured_log(logger, event, level=logging.INFO, **fields):
_carrier, trace_id, span_id = current_trace_fields()
payload = {
"event": event,
"service.name": os.environ.get("OTEL_SERVICE_NAME", "messenger"),
}
if trace_id:
payload["trace_id"] = trace_id
if span_id:
payload["span_id"] = span_id
for key, value in fields.items():
if value is None:
continue
if key in ("password", "token", "authorization", "body", "text", "bytes"):
continue
payload[key] = value
logger.log(level, json.dumps(payload, ensure_ascii=False, sort_keys=True))
@contextmanager
def start_span(name, kind=SpanKind.INTERNAL, attributes=None, links=None, context=None):
attrs = dict(attributes or {})
span = tracer().start_span(name, context=context, kind=kind, attributes=attrs, links=links or [])
token = trace.use_span(span, end_on_exit=True)
token.__enter__()
try:
yield span
except Exception as exc:
span.record_exception(exc)
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
raise
finally:
token.__exit__(None, None, None)
def link_from_carrier(carrier):
ctx = extract_context(carrier)
span_ctx = trace.get_current_span(ctx).get_span_context()
if span_ctx.is_valid:
return [Link(span_ctx)]
return []
def observe_process():
try:
import resource
usage = resource.getrusage(resource.RUSAGE_SELF)
_STATE["process_rss"] = float(usage.ru_maxrss) * 1024.0
_STATE["process_cpu"] = float(usage.ru_utime + usage.ru_stime)
except Exception:
pass
def set_pool_stats(used, waiting):
_STATE["db_pool_used"] = float(used)
_STATE["db_pool_waiting"] = float(waiting)
def set_worker_stats(busy, capacity):
_STATE["worker_busy"] = float(busy)
_STATE["worker_capacity"] = float(capacity)
def set_queue_stats(name, depth, oldest_age_s):
_STATE["queue_depth"][name] = int(depth)
_STATE["oldest_job_age"][name] = float(oldest_age_s)
def record_job(outcome, duration_s, job_type="attachment.process.v1"):
if jobs_processed is None:
return
attrs = {"job.type": job_type, "job.outcome": outcome}
jobs_processed.add(1, attrs)
job_duration.record(duration_s, attrs)
def record_time_to_ready(seconds):
if e2e_ready is None:
return
e2e_ready.record(float(seconds), {"job.type": "attachment.process.v1"})
def now_monotonic():
return time.perf_counter()
+241
View File
@@ -0,0 +1,241 @@
"""Отдельный процесс. API не уменьшает картинку, даже если worker остановлен."""
import logging
import os
import socket
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import pika
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from blobs import bucket_ready, read_secret
from broker import PROCESS_QUEUE, RETRY_QUEUE, _parameters, broker_channel, ensure_topology, publish_pending, queue_depth
from jobs import cleanup_staging, lease_seconds, process_delivery, release_expired
from telemetry import observe_process, set_queue_stats, set_worker_stats, setup as setup_telemetry, structured_log
LOGGER = logging.getLogger("worker")
SCHEMA_VERSION = "004_jobs"
BUSY = 0
BUSY_LOCK = threading.Lock()
def database_url():
return read_secret(os.environ["DATABASE_URL_FILE"])
def concurrency():
value = int(os.environ.get("WORKER_CONCURRENCY", "2"))
if not 1 <= value <= 8:
raise RuntimeError("WORKER_CONCURRENCY")
return value
def wait_schema(conninfo):
last = "error"
for _ in range(90):
try:
with psycopg.connect(conninfo, connect_timeout=3, row_factory=dict_row) as conn:
row = conn.execute(
"SELECT 1 FROM schema_migrations WHERE version = %s",
(SCHEMA_VERSION,),
).fetchone()
if row is not None:
return
last = "migration missing"
except psycopg.Error as exc:
last = type(exc).__name__
time.sleep(1)
LOGGER.error("schema not ready: %s", last)
raise SystemExit(1)
def wait_bucket():
for _ in range(60):
if bucket_ready():
return
time.sleep(1)
LOGGER.error("object storage not ready")
raise SystemExit(1)
def wait_broker():
last = "error"
for _ in range(60):
try:
with broker_channel():
return
except Exception as exc:
last = type(exc).__name__
time.sleep(1)
LOGGER.error("broker not ready: %s", last)
raise SystemExit(1)
def make_pool(conninfo, slots):
return ConnectionPool(
conninfo,
min_size=1,
max_size=slots * 2 + 2,
timeout=3,
kwargs={
"connect_timeout": 3,
"row_factory": dict_row,
"application_name": "mtusi-lab6-worker",
"options": "-c statement_timeout=5000 -c lock_timeout=3000 -c idle_in_transaction_session_timeout=30000",
},
open=True,
)
def oldest_job_age_seconds(pool):
try:
with pool.connection() as conn:
row = conn.execute(
"""
SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) AS age
FROM attachment_jobs
WHERE status IN ('queued', 'processing')
"""
).fetchone()
if row is None or row["age"] is None:
return 0.0
return float(row["age"])
except Exception:
return 0.0
def refresh_runtime_stats(pool, slots):
global BUSY
try:
depth = queue_depth(PROCESS_QUEUE)
except Exception:
depth = 0
try:
retry_depth = queue_depth(RETRY_QUEUE)
except Exception:
retry_depth = 0
age = oldest_job_age_seconds(pool)
set_queue_stats(PROCESS_QUEUE, depth, age)
set_queue_stats(RETRY_QUEUE, retry_depth, 0.0)
with BUSY_LOCK:
busy = BUSY
set_worker_stats(busy, slots)
observe_process()
try:
stats = pool.get_stats()
from telemetry import set_pool_stats
used = int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0))
waiting = int(stats.get("requests_waiting", 0))
set_pool_stats(max(0, used), waiting)
except Exception:
pass
def consume(pool, owner, slots):
executor = ThreadPoolExecutor(max_workers=slots, thread_name_prefix="job")
while True:
connection = None
try:
connection = pika.BlockingConnection(_parameters())
channel = connection.channel()
ensure_topology(channel)
channel.basic_qos(prefetch_count=slots)
def on_message(ch, method, _props, body):
executor.submit(run_one, pool, owner, ch, method.delivery_tag, body)
channel.basic_consume(PROCESS_QUEUE, on_message, auto_ack=False)
channel.start_consuming()
except Exception as exc:
LOGGER.warning("consumer %s", type(exc).__name__)
time.sleep(1)
finally:
if connection is not None:
try:
connection.close()
except Exception:
pass
def run_one(pool, owner, channel, tag, body):
global BUSY
with BUSY_LOCK:
BUSY += 1
try:
process_delivery(pool, body, owner)
action = "ack"
except Exception as exc:
LOGGER.warning("delivery %s", type(exc).__name__)
action = "nack"
finally:
with BUSY_LOCK:
BUSY = max(0, BUSY - 1)
def finish():
try:
if action == "ack":
channel.basic_ack(tag)
else:
channel.basic_nack(tag, requeue=True)
except Exception:
pass
try:
channel.connection.add_callback_threadsafe(finish)
except Exception:
pass
def main():
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker"))
conninfo = database_url()
wait_schema(conninfo)
wait_bucket()
wait_broker()
slots = concurrency()
pool = make_pool(conninfo, slots)
owner = socket.gethostname()
stop = threading.Event()
set_worker_stats(0, slots)
def drain():
while not stop.wait(1):
try:
with pool.connection() as conn:
publish_pending(conn)
except Exception as exc:
LOGGER.warning("drain %s", type(exc).__name__)
def sweep():
ticks = 0
while not stop.wait(5):
ticks += 1
try:
with pool.connection() as conn:
release_expired(conn)
if ticks % 12 == 0:
cleanup_staging(conn)
refresh_runtime_stats(pool, slots)
except Exception as exc:
LOGGER.warning("sweep %s", type(exc).__name__)
threading.Thread(target=drain, name="drain", daemon=True).start()
threading.Thread(target=sweep, name="sweep", daemon=True).start()
try:
with pool.connection() as conn:
publish_pending(conn)
except Exception as exc:
LOGGER.warning("startup drain %s", type(exc).__name__)
structured_log(LOGGER, "worker_ready", owner=owner, concurrency=slots, lease=lease_seconds())
LOGGER.info("worker ready owner=%s concurrency=%s lease=%s", owner, slots, lease_seconds())
consume(pool, owner, slots)
if __name__ == "__main__":
main()
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
import json
import subprocess
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args):
return subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=str(ROOT),
capture_output=True,
text=True,
)
def main():
try:
urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read()
except urllib.error.HTTPError:
pass
time.sleep(10)
end = int(time.time())
start = end - 1800
for query in (
'{service_name="messenger-api"}',
'{service_name="messenger-worker"}',
'{service_name=~".+"}',
):
url = "http://127.0.0.1:3100/loki/api/v1/query_range?" + urllib.parse.urlencode(
{
"query": query,
"start": str(start) + "000000000",
"end": str(end) + "000000000",
"limit": "20",
}
)
out = compose("exec", "-T", "loki", "wget", "-qO-", url)
try:
data = json.loads(out.stdout)
streams = data.get("data", {}).get("result", [])
lines = sum(len(item.get("values", [])) for item in streams)
print(query, "streams", len(streams), "lines", lines)
if streams and streams[0].get("values"):
print("sample", streams[0]["values"][0][1][:240])
except Exception as exc:
print(query, "err", exc, out.stdout[:200], out.stderr[:200])
print("collector tail:")
print(compose("logs", "otel-collector", "--since", "2m").stdout[-2000:])
q = '{service_name="messenger-api"} |= "http_request"'
url = "http://127.0.0.1:3100/loki/api/v1/query_range?" + urllib.parse.urlencode(
{
"query": q,
"start": str(start) + "000000000",
"end": str(end) + "000000000",
"limit": "10",
}
)
out = compose("exec", "-T", "loki", "wget", "-qO-", url)
data = json.loads(out.stdout)
streams = data.get("data", {}).get("result", [])
print("http_request streams", len(streams), "lines", sum(len(s.get("values", [])) for s in streams))
for stream in streams[:3]:
for value in stream.get("values", [])[:2]:
print(value[1][:400])
if __name__ == "__main__":
main()
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env python3
import base64
import json
import subprocess
import time
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = "http://127.0.0.1:8080"
def compose(*args):
return subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=str(ROOT),
capture_output=True,
text=True,
)
def http(method, path, body=None, headers=None):
data = None if body is None else json.dumps(body).encode()
req = urllib.request.Request(BASE + path, data=data, method=method, headers=headers or {})
if data is not None:
req.add_header("Content-Type", "application/json")
with urllib.request.urlopen(req, timeout=30) as resp:
raw = resp.read()
payload = json.loads(raw.decode()) if raw else None
return resp.status, dict(resp.headers), payload
def main():
stamp = uuid.uuid4().hex[:10]
users = []
for idx in range(2):
username = f"tr_{stamp}_{idx}"
http("POST", "/api/v1/users", {"username": username, "display_name": username, "password": "Password123!"})
auth = "Basic " + base64.b64encode(f"{username}:Password123!".encode()).decode()
_status, _headers, session = http("POST", "/api/v1/auth/sessions", headers={"Authorization": auth})
users.append(session)
_status, _headers, chat = http(
"POST",
"/api/v1/chats",
{"title": "t", "member_ids": [users[0]["user"]["id"], users[1]["user"]["id"]]},
headers={"Authorization": "Bearer " + users[0]["token"]},
)
png = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
req = urllib.request.Request(
BASE + f"/api/v1/chats/{chat['id']}/attachments?filename=t.png",
data=png,
method="POST",
headers={"Authorization": "Bearer " + users[0]["token"], "Content-Type": "image/png"},
)
with urllib.request.urlopen(req) as resp:
attachment = json.loads(resp.read().decode())
request_id = resp.headers.get("X-Request-Id")
print("upload", resp.status, attachment["id"], request_id)
for _ in range(60):
_status, _headers, payload = http(
"GET",
f"/api/v1/attachments/{attachment['id']}",
headers={"Authorization": "Bearer " + users[0]["token"]},
)
if payload.get("status") == "ready":
print("ready")
break
time.sleep(0.5)
time.sleep(6)
for service in ("messenger-api", "messenger-worker"):
q = urllib.parse.urlencode({"tags": f"service.name={service}", "limit": "20"})
out = compose("exec", "-T", "tempo", "wget", "-qO-", f"http://127.0.0.1:3200/api/search?{q}")
print(service, out.stdout[:800])
q = urllib.parse.urlencode({"query": '{service_name="messenger-api"} |= "http_request"', "limit": "5"})
out = compose("exec", "-T", "loki", "wget", "-qO-", "http://127.0.0.1:3100/loki/api/v1/query?" + q)
print("loki", out.stdout[:600])
# worker stop age probe
compose("stop", "worker")
req = urllib.request.Request(
BASE + f"/api/v1/chats/{chat['id']}/attachments?filename=lag.png",
data=png,
method="POST",
headers={"Authorization": "Bearer " + users[0]["token"], "Content-Type": "image/png"},
)
with urllib.request.urlopen(req) as resp:
queued = json.loads(resp.read().decode())
print("queued", resp.status, queued["id"])
ages = []
fired = None
started = time.time()
while time.time() - started < 150:
q = urllib.parse.urlencode(
{
"query": 'max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"})'
}
)
out = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/query?" + q)
age = None
try:
payload = json.loads(out.stdout)
rows = payload.get("data", {}).get("result", [])
if rows:
age = float(rows[0]["value"][1])
except Exception:
age = None
ages.append({"t": round(time.time() - started, 1), "age": age})
alerts = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/alerts")
try:
alert_payload = json.loads(alerts.stdout)
for item in alert_payload.get("data", {}).get("alerts", []):
if item.get("labels", {}).get("alertname") == "GrowingQueueLag" and item.get("state") == "firing":
fired = round(time.time() - started, 1)
break
except Exception:
pass
if fired is not None:
break
time.sleep(5)
compose("start", "worker")
print(json.dumps({"ages": ages[-8:], "fired_after_seconds": fired}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+136
View File
@@ -0,0 +1,136 @@
"""Негативные проверки API. Токены и пароли не печатаются."""
import json
import struct
import sys
import uuid
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def png(width, height, payload=b""):
def chunk(tag, data):
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + payload + chunk(b"IEND", b"")
def main():
base = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8080"
client = Client(base)
alice = client.user("n1")
bob = client.user("n2")
eve = client.user("n3")
chat = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Негативные", "member_ids": [alice["id"].upper(), bob["id"]]},
schema="Chat",
)["json"]
other = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Другой", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
upper = client.call("GET", "/api/v1/chats/" + chat["id"].upper(), actor=bob, schema="Chat")["json"]
results = []
def record(name, status):
results.append({"name": name, "status": status})
record("chat_uuid_case", 200 if upper["id"] == chat["id"] else 0)
path = "/api/v1/chats/" + chat["id"] + "/messages"
record("blank_text", client.call("POST", path, 400, actor=alice, body={"text": " "}, headers={"Idempotency-Key": "blank-1"})["status"])
record("bad_json", client.call("POST", path, 400, actor=alice, raw=b"{", headers={"Content-Type": "application/json", "Idempotency-Key": "bad-json"})["status"])
record("short_password", client.call("POST", "/api/v1/users", 400, body={"username": "abc", "display_name": "Имя", "password": "short"})["status"])
key = uuid.uuid4().hex
first = client.call("POST", path, 201, actor=alice, body={"text": "одно"}, headers={"Idempotency-Key": key}, schema="Message")["json"]
second = client.call(
"POST",
path,
201,
actor=alice,
body={"text": "одно", "attachment_ids": []},
headers={"Idempotency-Key": key},
schema="Message",
)["json"]
record("omitted_equals_empty", 201 if first == second else 0)
record("same_key_other_body", client.call("POST", path, 409, actor=alice, body={"text": "другое"}, headers={"Idempotency-Key": key})["status"])
history = client.call("GET", path.replace("/messages", "/messages?limit=100"), actor=bob, schema="MessagePage")["json"]
record("no_second_row", 200 if sum(1 for item in history["items"] if item["text"] in ("одно", "другое")) == 1 else 0)
other_path = "/api/v1/chats/" + other["id"] + "/messages"
other_message = client.call(
"POST",
other_path,
201,
actor=alice,
body={"text": "одно"},
headers={"Idempotency-Key": key},
schema="Message",
)["json"]
record("same_key_other_chat", 201 if other_message["id"] != first["id"] else 0)
record("foreign_chat", client.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)["status"])
record("bad_cursor", client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?cursor=not-a-cursor", 400, actor=alice)["status"])
upload = "/api/v1/chats/" + chat["id"] + "/attachments"
record("slash_name", client.call("POST", upload + "?filename=a/b.png", 400, actor=alice, raw=png(1, 1), headers={"Content-Type": "image/png"})["status"])
record("huge_pixels", client.call("POST", upload + "?filename=big.png", 413, actor=alice, raw=png(8000, 8000), headers={"Content-Type": "image/png"})["status"])
record("bad_png", client.call("POST", upload + "?filename=bad.png", 415, actor=alice, raw=b"\x89PNG\r\n\x1a\nbroken", headers={"Content-Type": "image/png"})["status"])
record("type_mismatch", client.call("POST", upload + "?filename=no.jpg", 415, actor=alice, raw=png(1, 1), headers={"Content-Type": "image/jpeg"})["status"])
record("oversize", client.call("POST", upload + "?filename=over.png", 413, actor=alice, raw=b"x" * (10 * 1024 * 1024 + 1), headers={"Content-Type": "image/png"})["status"])
missing = str(uuid.uuid4())
record("unknown_attachment", client.call("GET", "/api/v1/attachments/" + missing.upper(), 404, actor=alice)["status"])
record("no_token", client.call("GET", "/api/v1/users/me", 401)["status"])
payload = {
"command": "python -X utf8 student-tests/run_negative.py",
"result": "PASS negative",
"results": results,
}
body = json.dumps(payload, ensure_ascii=False, indent=2)
print(body)
failed = [item for item in results if item["status"] not in (200, 201, 400, 401, 404, 409, 413, 415)]
if failed:
return 1
expected = {
"chat_uuid_case": 200,
"blank_text": 400,
"bad_json": 400,
"short_password": 400,
"omitted_equals_empty": 201,
"same_key_other_body": 409,
"no_second_row": 200,
"same_key_other_chat": 201,
"foreign_chat": 404,
"bad_cursor": 400,
"slash_name": 400,
"huge_pixels": 413,
"bad_png": 415,
"type_mismatch": 415,
"oversize": 413,
"unknown_attachment": 404,
"no_token": 401,
}
for name, status in expected.items():
got = next(item["status"] for item in results if item["name"] == name)
if got != status:
print("mismatch " + name, file=sys.stderr)
return 1
(ROOT / "evidence" / "negative.json").write_text(body + "\n", encoding="utf-8")
print("PASS negative")
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except Failure as exc:
print("FAIL " + str(exc), file=sys.stderr)
raise SystemExit(1)
+474
View File
@@ -0,0 +1,474 @@
#!/usr/bin/env python3
"""Учебные проверки наблюдаемости и инцидентов. Пароли не печатает."""
from __future__ import annotations
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
OUT = ROOT / "evidence" / "observability.json"
def http_json(method, path, body=None, headers=None, timeout=30):
data = None if body is None else json.dumps(body).encode("utf-8")
req = urllib.request.Request(
BASE + path,
data=data,
method=method,
headers=headers or {},
)
if data is not None:
req.add_header("Content-Type", "application/json")
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read()
payload = json.loads(raw.decode("utf-8")) if raw else None
return resp.status, dict(resp.headers), payload
except urllib.error.HTTPError as exc:
raw = exc.read()
payload = None
if raw:
try:
payload = json.loads(raw.decode("utf-8"))
except ValueError:
payload = {"raw": raw.decode("utf-8", errors="replace")[:200]}
return exc.code, dict(exc.headers), payload
def compose(*args):
cmd = ["docker", "compose", "--env-file", ".env", *args]
return subprocess.run(cmd, cwd=str(ROOT), capture_output=True, text=True, check=False)
def prom_query(expr):
q = urllib.parse.urlencode({"query": expr})
url = "http://127.0.0.1:9090/api/v1/query?" + q
# Prometheus не опубликован наружу. Читаем через docker exec.
result = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/query?" + q)
if result.returncode != 0:
return None
try:
payload = json.loads(result.stdout)
except ValueError:
return None
rows = payload.get("data", {}).get("result", [])
if not rows:
return 0.0
return float(rows[0]["value"][1])
def prom_alerts():
result = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/alerts")
if result.returncode != 0:
return []
try:
payload = json.loads(result.stdout)
except ValueError:
return []
return payload.get("data", {}).get("alerts", [])
def tempo_search(service, name=None, limit=10):
# Tag search + TraceQL fallback.
tags = "service.name=" + service
q = urllib.parse.urlencode({"tags": tags, "limit": str(limit)})
result = compose("exec", "-T", "tempo", "wget", "-qO-", "http://127.0.0.1:3200/api/search?" + q)
traces = []
if result.returncode == 0:
try:
traces = json.loads(result.stdout).get("traces", [])
except ValueError:
traces = []
if name:
traces = [t for t in traces if name in (t.get("rootTraceName") or "")]
if traces:
return traces
query = f'{{resource.service.name="{service}"}}'
if name:
query = f'{{resource.service.name="{service}" && name="{name}"}}'
q = urllib.parse.urlencode({"q": query, "limit": str(limit)})
result = compose("exec", "-T", "tempo", "wget", "-qO-", "http://127.0.0.1:3200/api/search?" + q)
if result.returncode != 0:
return []
try:
return json.loads(result.stdout).get("traces", [])
except ValueError:
return []
def register_and_chat():
stamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S%f")
users = []
for idx in range(2):
username = f"obs_{stamp}_{idx}"
status, _, user = http_json(
"POST",
"/api/v1/users",
{"username": username, "display_name": username, "password": "Password123!"},
)
assert status == 201, status
status, _, session = http_json(
"POST",
"/api/v1/auth/sessions",
headers={"Authorization": "Basic " + __import__("base64").b64encode(f"{username}:Password123!".encode()).decode()},
)
assert status == 201, status
users.append({"user": user, "token": session["token"]})
status, _, chat = http_json(
"POST",
"/api/v1/chats",
{"title": "obs", "member_ids": [users[0]["user"]["id"], users[1]["user"]["id"]]},
headers={"Authorization": "Bearer " + users[0]["token"]},
)
assert status == 201, status
return users, chat
def upload_png(token, chat_id, name="probe.png"):
data = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
req = urllib.request.Request(
f"{BASE}/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(name)}",
data=data,
method="POST",
headers={
"Authorization": "Bearer " + token,
"Content-Type": "image/png",
"Content-Length": str(len(data)),
},
)
with urllib.request.urlopen(req, timeout=30) as resp:
payload = json.loads(resp.read().decode("utf-8"))
return resp.status, dict(resp.headers), payload
def api_direct(method, path, token=None, timeout=3):
"""Запрос внутрь контейнера API, минуя HAProxy. Нужен, когда ready уже красный."""
headers = ["-H", "Accept: application/json"]
if token:
headers.extend(["-H", f"Authorization: Bearer {token}"])
cmd = [
"exec",
"-T",
"api",
"python",
"-c",
(
"import os,urllib.request,sys\n"
f"req=urllib.request.Request('http://127.0.0.1:8080{path}', method='{method}')\n"
),
]
# Проще через wget/python one-liner in api container.
py = (
"import urllib.request,urllib.error\n"
f"req=urllib.request.Request('http://127.0.0.1:8080{path}', method='{method}')\n"
)
if token:
py += f"req.add_header('Authorization', 'Bearer {token}')\n"
py += (
"try:\n"
f" r=urllib.request.urlopen(req, timeout={timeout})\n"
" print(r.status)\n"
"except urllib.error.HTTPError as e:\n"
" print(e.code)\n"
"except Exception as e:\n"
" print(type(e).__name__)\n"
)
result = compose("exec", "-T", "api", "python", "-c", py)
text = (result.stdout or "").strip().splitlines()
return text[-1] if text else "empty"
def wait_ready_attachment(token, attachment_id, timeout=60):
deadline = time.time() + timeout
while time.time() < deadline:
status, _, payload = http_json(
"GET",
f"/api/v1/attachments/{attachment_id}",
headers={"Authorization": "Bearer " + token},
)
if status == 200 and payload.get("status") == "ready":
return payload
time.sleep(0.5)
raise TimeoutError(attachment_id)
def scenario_trace_link():
users, chat = register_and_chat()
token = users[0]["token"]
status, headers, attachment = upload_png(token, chat["id"])
assert status == 202, status
ready = wait_ready_attachment(token, attachment["id"])
time.sleep(3)
api_traces = tempo_search("messenger-api", "POST /api/v1/chats/{chat_id}/attachments")
worker_traces = tempo_search("messenger-worker", "attachments.process consume")
return {
"ok": True,
"attachment_id": attachment["id"],
"attachment_status": ready["status"],
"request_id": headers.get("X-Request-Id") or headers.get("x-request-id"),
"api_upload_traces": len(api_traces),
"worker_consume_traces": len(worker_traces),
"sample_api_trace": api_traces[0]["traceID"] if api_traces else None,
"sample_worker_trace": worker_traces[0]["traceID"] if worker_traces else None,
}
def scenario_worker_stop():
users, chat = register_and_chat()
token = users[0]["token"]
compose("stop", "worker")
time.sleep(2)
status, _, attachment = upload_png(token, chat["id"], "queued.png")
assert status == 202, status
started = time.time()
fired_at = None
age_samples = []
while time.time() - started < 90:
age = prom_query('max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"})')
age_samples.append({"t": round(time.time() - started, 1), "age": age})
alerts = [a for a in prom_alerts() if a.get("labels", {}).get("alertname") == "GrowingQueueLag" and a.get("state") == "firing"]
if alerts and fired_at is None:
fired_at = round(time.time() - started, 1)
break
time.sleep(5)
compose("start", "worker")
ready = wait_ready_attachment(token, attachment["id"], timeout=120)
resolved_at = None
recover_started = time.time()
while time.time() - recover_started < 120:
alerts = [a for a in prom_alerts() if a.get("labels", {}).get("alertname") == "GrowingQueueLag" and a.get("state") == "firing"]
if not alerts and fired_at is not None:
resolved_at = round(time.time() - recover_started, 1)
break
time.sleep(5)
return {
"ok": ready["status"] == "ready",
"attachment_id": attachment["id"],
"fired_after_seconds": fired_at,
"resolved_after_seconds": resolved_at,
"age_samples": age_samples[-6:],
"final_age": prom_query('messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"}'),
}
def scenario_db_stop():
# Отказ Postgres. Через HAProxy ready красный, поэтому бизнес-5xx снимаю прямым
# запросом в контейнер API, чтобы метрики и traces остались от приложения.
users, chat = register_and_chat()
token = users[0]["token"]
before_5xx = prom_query('sum(messenger_http_requests_total{http_response_status_class="5xx"}) or vector(0)')
compose("stop", "postgres")
started = time.time()
business_codes = []
ready_via_lb = []
while time.time() - started < 90:
business_codes.append(api_direct("GET", "/api/v1/users/me", token=token))
try:
ready_status, _, _ = http_json("GET", "/health/ready", timeout=3)
except Exception as exc:
ready_status = type(exc).__name__
ready_via_lb.append(ready_status)
time.sleep(1)
high_fired = None
for sample_t in range(0, 90, 5):
pass
compose("start", "postgres")
deadline = time.time() + 90
while time.time() < deadline:
try:
status, _, payload = http_json("GET", "/health/ready", timeout=3)
if status == 200 and payload and payload.get("status") == "ok":
break
except Exception:
pass
time.sleep(2)
# ждём for: 1m у HighHttpErrorRate и запись метрик
watch_started = time.time()
alerts_timeline = []
while time.time() - watch_started < 150:
alerts = [
a
for a in prom_alerts()
if a.get("labels", {}).get("alertname")
in ("HighHttpErrorRate", "ErrorBudgetBurnFast", "ErrorBudgetBurnSlow")
]
alerts_timeline.append(
{
"t": round(time.time() - watch_started, 1),
"alerts": [
{"alertname": a.get("labels", {}).get("alertname"), "state": a.get("state")}
for a in alerts
],
}
)
if any(a.get("state") == "firing" for a in alerts) and high_fired is None:
high_fired = round(time.time() - watch_started, 1)
if high_fired is not None and not any(a.get("state") == "firing" for a in alerts):
break
time.sleep(5)
after_5xx = prom_query('sum(messenger_http_requests_total{http_response_status_class="5xx"}) or vector(0)')
return {
"ok": True,
"business_statuses_direct_api": business_codes[:8] + ["..."] + business_codes[-4:],
"ready_via_lb": ready_via_lb[:4] + ["..."] + ready_via_lb[-3:],
"http_5xx_before": before_5xx,
"http_5xx_after": after_5xx,
"high_error_seen_after_seconds": high_fired,
"alerts_timeline_tail": alerts_timeline[-6:],
"note": "Прямые запросы в api-контейнер, потому что HAProxy при красном ready отдаёт свою 503-страницу",
}
def scenario_telemetry_backend_down():
users, chat = register_and_chat()
token = users[0]["token"]
compose("stop", "tempo")
compose("stop", "loki")
compose("stop", "prometheus")
time.sleep(2)
mem_before = compose("stats", "--no-stream", "--format", "{{.MemUsage}}", "otel-collector").stdout.strip()
status, _, attachment = upload_png(token, chat["id"], "tel.png")
assert status == 202, status
# API должен жить без telemetry backends
ready_status, _, ready_body = http_json("GET", "/health/ready")
text_status, _, message = http_json(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
{"text": "telemetry backends down"},
headers={
"Authorization": "Bearer " + token,
"Idempotency-Key": "obs-tel-1",
},
)
time.sleep(8)
mem_after = compose("stats", "--no-stream", "--format", "{{.MemUsage}}", "otel-collector").stdout.strip()
compose("start", "prometheus")
compose("start", "loki")
compose("start", "tempo")
time.sleep(5)
compose("restart", "otel-collector")
wait_ready_attachment(token, attachment["id"], timeout=120)
return {
"ok": ready_status == 200 and text_status == 201,
"upload_status": status,
"ready_status": ready_status,
"message_status": text_status,
"message_id": None if not message else message.get("id"),
"otel_mem_before": mem_before,
"otel_mem_after": mem_after,
}
def scenario_burn_rate_spike():
# Короткий всплеск и затем более длинная деградация для multi-window burn-rate.
users, _chat = register_and_chat()
token = users[0]["token"]
def outage(seconds):
compose("stop", "postgres")
end = time.time() + seconds
while time.time() < end:
api_direct("GET", "/api/v1/users/me", token=token)
time.sleep(0.5)
compose("start", "postgres")
deadline = time.time() + 90
while time.time() < deadline:
try:
status, _, payload = http_json("GET", "/health/ready", timeout=3)
if status == 200 and payload and payload.get("status") == "ok":
return
except Exception:
pass
time.sleep(2)
# краткий всплеск ~25 с
brief_started = time.time()
outage(25)
brief_high = None
brief_burn = None
watch = time.time()
while time.time() - watch < 120:
alerts = prom_alerts()
if brief_high is None and any(
a.get("labels", {}).get("alertname") == "HighHttpErrorRate" and a.get("state") == "firing" for a in alerts
):
brief_high = round(time.time() - brief_started, 1)
if brief_burn is None and any(
a.get("labels", {}).get("alertname") == "ErrorBudgetBurnFast" and a.get("state") == "firing" for a in alerts
):
brief_burn = round(time.time() - brief_started, 1)
if brief_high is not None:
break
time.sleep(5)
# длительная деградация ~6 минут
long_started = time.time()
outage(360)
long_slow = None
long_fast = None
watch = time.time()
while time.time() - watch < 240:
alerts = prom_alerts()
if long_slow is None and any(
a.get("labels", {}).get("alertname") == "ErrorBudgetBurnSlow" and a.get("state") == "firing" for a in alerts
):
long_slow = round(time.time() - long_started, 1)
if long_fast is None and any(
a.get("labels", {}).get("alertname") == "ErrorBudgetBurnFast" and a.get("state") == "firing" for a in alerts
):
long_fast = round(time.time() - long_started, 1)
if long_slow is not None:
break
time.sleep(5)
return {
"ok": True,
"brief_spike_seconds": 25,
"brief_high_error_fired_after_seconds": brief_high,
"brief_burn_fast_fired_after_seconds": brief_burn,
"long_degradation_seconds": 360,
"long_burn_slow_fired_after_seconds": long_slow,
"long_burn_fast_fired_after_seconds": long_fast,
"error_ratio_5m": prom_query("messenger:http_error_ratio:5m"),
"error_ratio_30m": prom_query("messenger:http_error_ratio:30m"),
"note": "Короткий всплеск может не зажечь BurnFast (нет ложного page). Длинная деградация должна зажечь BurnSlow.",
}
def main():
report = {
"started_at": datetime.now(timezone.utc).isoformat(),
"base_url": BASE,
"scenarios": {},
}
for name, func in [
("trace_link", scenario_trace_link),
("worker_stop", scenario_worker_stop),
("db_stop", scenario_db_stop),
("telemetry_backend_down", scenario_telemetry_backend_down),
("burn_rate_spike", scenario_burn_rate_spike),
]:
print("RUN", name, flush=True)
try:
report["scenarios"][name] = func()
print("OK", name, flush=True)
except Exception as exc:
report["scenarios"][name] = {"ok": False, "error": type(exc).__name__, "detail": str(exc)[:300]}
print("FAIL", name, type(exc).__name__, flush=True)
report["finished_at"] = datetime.now(timezone.utc).isoformat()
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("WROTE", OUT)
failed = [k for k, v in report["scenarios"].items() if not v.get("ok")]
return 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())
+653
View File
@@ -0,0 +1,653 @@
"""Проверки очереди и трека 2. Токены и пароли не печатаются."""
import hashlib
import json
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
from image_probe import jpeg_size # noqa: E402
BASE = "http://127.0.0.1:8080"
EVIDENCE = ROOT / "evidence" / "pipeline.json"
INSIDE = r"""
import json, logging, os, hashlib
from datetime import datetime, timezone
logging.disable(logging.CRITICAL)
req = json.loads(REQUEST_JSON)
from blobs import read_secret, read_bytes, put_bytes, delete_quiet
from jobs import bump_pipeline, replay_job, republish
import psycopg
from psycopg.rows import dict_row
url = read_secret(os.environ["DATABASE_URL_FILE"])
action = req["action"]
out = {"action": action}
with psycopg.connect(url, row_factory=dict_row) as conn:
if action == "republish":
out["ok"] = bool(republish(conn, req["job_id"], req.get("attempt")))
elif action == "bump":
out["result"] = bump_pipeline(conn, req["attachment_id"])
elif action == "replay":
out["result"] = replay_job(conn, req["job_id"])
elif action == "corrupt":
row = conn.execute(
"SELECT staging_key FROM attachments WHERE id = %s",
(req["attachment_id"],),
).fetchone()
key = row["staging_key"]
data = read_bytes(key)
stamp = datetime.now(timezone.utc)
put_bytes(key + ".held", data, "application/octet-stream", stamp)
put_bytes(key, b"not-an-image", "image/png", stamp)
out["original_sha256"] = hashlib.sha256(data).hexdigest()
out["bytes"] = len(data)
elif action == "restore":
row = conn.execute(
"SELECT staging_key FROM attachments WHERE id = %s",
(req["attachment_id"],),
).fetchone()
key = row["staging_key"]
data = read_bytes(key + ".held")
put_bytes(key, data, "image/png", datetime.now(timezone.utc))
delete_quiet(key + ".held")
out["restored_sha256"] = hashlib.sha256(data).hexdigest()
out["bytes"] = len(data)
else:
raise SystemExit("unknown")
print(json.dumps(out))
"""
def run(args, timeout=180, check=True):
proc = subprocess.run(
args,
cwd=str(ROOT),
capture_output=True,
timeout=timeout,
)
if check and proc.returncode != 0:
err = proc.stderr.decode("utf-8", "replace")[-1200:]
raise RuntimeError(args[0] + " failed: " + err)
return proc
def compose(*args, timeout=180):
run(["docker", "compose", "--env-file", ".env", *args], timeout=timeout)
def inspect(name, template):
proc = run(["docker", "inspect", "-f", template, name], timeout=30)
return proc.stdout.decode().strip()
def wait_health(name, seconds=180):
deadline = time.monotonic() + seconds
last = ""
while time.monotonic() < deadline:
last = inspect(name, "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}")
if last.startswith("running healthy"):
return last
time.sleep(1)
raise RuntimeError(name + " not healthy: " + last)
def restart_service(service, containers, seconds=180):
if isinstance(containers, str):
containers = [containers]
before = {name: inspect(name, "{{.State.StartedAt}}") for name in containers}
compose("restart", service)
deadline = time.monotonic() + seconds
last = ""
while time.monotonic() < deadline:
ready = True
for name in containers:
started = inspect(name, "{{.State.StartedAt}}")
last = inspect(name, "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}")
if started == before[name] or not last.startswith("running healthy"):
ready = False
if ready:
return
time.sleep(1)
raise RuntimeError(service + " did not restart: " + last)
def psql(sql):
proc = run(
[
"docker", "compose", "--env-file", ".env", "exec", "-T", "postgres",
"psql", "-U", "messenger", "-d", "messenger",
"-v", "ON_ERROR_STOP=1", "-q", "-t", "-A", "-c", sql,
],
timeout=30,
)
return proc.stdout.decode("utf-8").strip()
def state(attachment_id):
sql = """
SELECT json_build_object(
'status', a.status,
'error_code', a.error_code,
'pipeline_version', a.pipeline_version,
'jobs', (
SELECT coalesce(json_agg(json_build_object(
'id', j.id, 'status', j.status, 'attempt', j.attempt,
'pipeline_version', j.pipeline_version, 'error_code', j.error_code
) ORDER BY j.pipeline_version), '[]'::json)
FROM attachment_jobs j WHERE j.attachment_id = a.id
),
'variants', (
SELECT coalesce(json_agg(json_build_object(
'name', v.name, 'pipeline_version', v.pipeline_version,
'width', v.width, 'height', v.height, 'content_type', v.content_type
) ORDER BY v.pipeline_version, v.name), '[]'::json)
FROM attachment_variants v WHERE v.attachment_id = a.id
),
'unpublished', (
SELECT count(*) FROM outbox o
JOIN attachment_jobs j ON j.id = o.job_id
WHERE j.attachment_id = a.id AND o.published_at IS NULL
),
'events', (
SELECT coalesce(json_agg(e.kind ORDER BY e.created_at), '[]'::json)
FROM job_events e WHERE e.attachment_id = a.id
)
)::text
FROM attachments a WHERE a.id = '{id}'::uuid
""".format(id=attachment_id)
return json.loads(psql(sql))
def arm(kind, attachment_id):
target = "NULL" if attachment_id is None else "'" + attachment_id + "'::uuid"
psql(
"INSERT INTO fault_injections (id, kind, attachment_id, armed) "
"VALUES (gen_random_uuid(), '{kind}', {target}, true)".format(kind=kind, target=target)
)
def queues():
proc = run(
[
"docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq",
"rabbitmqctl", "list_queues", "name", "messages", "--quiet",
],
timeout=30,
)
found = {}
for line in proc.stdout.decode().splitlines():
parts = line.split()
if len(parts) == 2 and parts[1].isdigit():
found[parts[0]] = int(parts[1])
return found
def depth():
last = ""
for _ in range(20):
try:
found = queues()
return found.get("attachments.process", -1)
except RuntimeError as exc:
last = str(exc)
time.sleep(1)
raise RuntimeError(last)
def inside(req, timeout=90):
code = INSIDE.replace("REQUEST_JSON", json.dumps(json.dumps(req)))
proc = run(
[
"docker", "compose", "--env-file", ".env", "run", "--rm", "--no-deps", "-T",
"--entrypoint", "python", "worker", "-c", code,
],
timeout=timeout,
)
lines = [line for line in proc.stdout.decode("utf-8").splitlines() if line.startswith("{")]
if not lines:
raise RuntimeError("no json from worker helper")
return json.loads(lines[-1])
def stop_worker():
compose("stop", "worker")
def start_worker():
compose("start", "worker")
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
if inspect("mtusi-lab5-worker-1", "{{.State.Status}}") == "running":
return
time.sleep(0.5)
raise RuntimeError("worker did not start")
def restart_count():
return int(inspect("mtusi-lab5-worker-1", "{{.RestartCount}}"))
def wait_status(actor, attachment_id, want, seconds=90):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
body = client.call(
"GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment"
)["json"]
last = body["status"]
if last == want:
return body
if want != "failed" and last == "failed":
raise Failure("attachment failed " + attachment_id)
time.sleep(0.4)
raise Failure("timeout status=" + str(last) + " want=" + want)
def check_picture(actor, ready, original):
download = client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=actor)
if download["bytes"] != original:
raise Failure("original bytes differ")
names = {item["name"]: item for item in ready["variants"]}
thumb = client.call(
"GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=thumbnail", actor=actor
)
crop = client.call(
"GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=crop", actor=actor
)
optimized = client.call(
"GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=optimized", actor=actor
)
if jpeg_size(thumb["bytes"]) != (256, 160):
raise Failure("thumbnail size")
if jpeg_size(crop["bytes"]) != (128, 128):
raise Failure("crop size")
if optimized["bytes"][:4] != b"RIFF" or optimized["bytes"][8:12] != b"WEBP":
raise Failure("optimized is not webp")
if names["thumbnail"]["width"] != 256 or names["thumbnail"]["height"] != 160:
raise Failure("thumbnail metadata")
if names["crop"]["width"] != 128 or names["crop"]["height"] != 128:
raise Failure("crop metadata")
if names["optimized"]["width"] != 320 or names["optimized"]["height"] != 200:
raise Failure("optimized metadata")
if hashlib.sha256(thumb["bytes"]).hexdigest() != names["thumbnail"]["sha256"]:
raise Failure("thumbnail sha")
rows = state(ready["id"])
version_rows = [row for row in rows["variants"] if row["pipeline_version"] == rows["pipeline_version"]]
if len(version_rows) != 4:
raise Failure("variant rows " + str(len(version_rows)))
return {
"pipeline_version": rows["pipeline_version"],
"variant_rows": len(version_rows),
"thumbnail": [names["thumbnail"]["width"], names["thumbnail"]["height"]],
"crop": [names["crop"]["width"], names["crop"]["height"]],
"optimized": [names["optimized"]["width"], names["optimized"]["height"]],
"original_sha256": hashlib.sha256(original).hexdigest(),
}
def upload(actor, chat):
att, content = client.attachment(actor, chat)
return att, content
results = []
def scenario(name, fn):
started = time.monotonic()
try:
detail = fn()
results.append({"name": name, "ok": True, "seconds": round(time.monotonic() - started, 3), "detail": detail})
except Exception as exc:
results.append({"name": name, "ok": False, "seconds": round(time.monotonic() - started, 3), "error": type(exc).__name__ + ": " + str(exc)})
raise
client = Client(BASE, timeout=20)
def durability():
stop_worker()
alice, bob, _eve, chat = client.bootstrap()
att, content = upload(alice, chat)
blocked = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
text = client.message(alice, chat, "текст пока файл в очереди")["json"]
linked = client.message(alice, chat, "", attachments=[att["id"]])["json"]
before_api = depth()
if before_api < 1:
raise Failure("job was not in the durable queue")
restart_service("api", ["mtusi-lab5-api-1", "mtusi-lab5-api-2"])
after_api = depth()
restart_service("rabbitmq", "mtusi-lab5-rabbitmq-1", 180)
after_rabbit = depth()
if after_api < 1 or after_rabbit < 1:
raise Failure("queue lost the job")
start_worker()
ready = wait_status(bob, att["id"], "ready", 90)
picture = check_picture(bob, ready, content)
return {
"upload_status": att["status"],
"content_while_queued": blocked["status"],
"text_status": 201,
"text_id": text["id"],
"linked_id": linked["id"],
"queue_before_api_restart": before_api,
"queue_after_api_restart": after_api,
"queue_after_rabbit_restart": after_rabbit,
"picture": picture,
}
def outbox_gap():
stop_worker()
alice, bob, _eve, chat = client.bootstrap()
arm("skip_inline_publish", None)
before = depth()
att, content = upload(alice, chat)
info = state(att["id"])
queued = depth()
if info["unpublished"] != 1 or queued != before:
raise Failure("outbox was published early")
start_worker()
ready = wait_status(bob, att["id"], "ready", 90)
picture = check_picture(bob, ready, content)
return {"unpublished_before_worker": info["unpublished"], "queue_depth": queued, "picture": picture}
def duplicate():
stop_worker()
alice, bob, _eve, chat = client.bootstrap()
att, content = upload(alice, chat)
info = state(att["id"])
job_id = info["jobs"][0]["id"]
published = inside({"action": "republish", "job_id": job_id})
start_worker()
ready = wait_status(bob, att["id"], "ready", 90)
picture = check_picture(bob, ready, content)
return {"republish": published["ok"], "picture": picture}
def reorder():
stop_worker()
alice, bob, _eve, chat = client.bootstrap()
att, content = upload(alice, chat)
info = state(att["id"])
job_id = info["jobs"][0]["id"]
inside({"action": "republish", "job_id": job_id, "attempt": 2})
start_worker()
ready = wait_status(bob, att["id"], "ready", 90)
picture = check_picture(bob, ready, content)
after = state(att["id"])
if after["jobs"][0]["attempt"] != 1 or after["status"] != "ready":
raise Failure("reorder changed the job")
return {"attempt": after["jobs"][0]["attempt"], "picture": picture}
def stale_version():
stop_worker()
alice, bob, _eve, chat = client.bootstrap()
att, content = upload(alice, chat)
info = state(att["id"])
old_job = info["jobs"][0]["id"]
bumped = inside({"action": "bump", "attachment_id": att["id"]})
start_worker()
ready = wait_status(bob, att["id"], "ready", 90)
before = check_picture(bob, ready, content)
thumb_before = client.call(
"GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob
)["bytes"]
inside({"action": "republish", "job_id": old_job, "attempt": 1})
time.sleep(2)
ready2 = client.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"]
thumb_after = client.call(
"GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob
)["bytes"]
after = state(att["id"])
old = [job for job in after["jobs"] if job["id"] == old_job][0]
if ready2["status"] != "ready" or after["pipeline_version"] != 2:
raise Failure("new version was overwritten")
if old["status"] != "superseded":
raise Failure("old job was not superseded")
if thumb_before != thumb_after:
raise Failure("old job changed thumbnail")
return {
"bump": bumped["result"],
"old_job_status": old["status"],
"old_error_code": old["error_code"],
"attachment_error_code": after["error_code"],
"picture": before,
"thumbnail_unchanged": True,
}
def crash(kind):
stop_worker()
alice, bob, _eve, chat = client.bootstrap()
att, content = upload(alice, chat)
arm(kind, att["id"])
start_worker()
born = inspect("mtusi-lab5-worker-1", "{{.State.StartedAt}}")
ready = wait_status(bob, att["id"], "ready", 120)
picture = check_picture(bob, ready, content)
restarted = inspect("mtusi-lab5-worker-1", "{{.State.StartedAt}}")
if born == restarted:
raise Failure("worker did not exit")
return {"started_at": born, "restarted_at": restarted, "picture": picture}
def transient():
alice, bob, _eve, chat = client.bootstrap()
stop_worker()
att, _content = upload(alice, chat)
arm("transient_once", att["id"])
start_worker()
wait_status(bob, att["id"], "ready", 90)
info = state(att["id"])
version_rows = [row for row in info["variants"] if row["pipeline_version"] == info["pipeline_version"]]
if info["jobs"][0]["attempt"] != 2 or "retry" not in info["events"] or len(version_rows) != 4:
raise Failure("transient retry did not keep one result")
return {
"attempt": info["jobs"][0]["attempt"],
"events": info["events"],
"variant_rows": len(version_rows),
"status": info["status"],
}
def permanent_and_replay():
stop_worker()
alice, bob, _eve, chat = client.bootstrap()
att, content = upload(alice, chat)
corrupted = inside({"action": "corrupt", "attachment_id": att["id"]})
start_worker()
failed = wait_status(bob, att["id"], "failed", 90)
info = state(att["id"])
restored = inside({"action": "restore", "attachment_id": att["id"]})
replayed = inside({"action": "replay", "job_id": info["jobs"][0]["id"]})
ready = wait_status(bob, att["id"], "ready", 90)
picture = check_picture(bob, ready, content)
again = inside({"action": "replay", "job_id": info["jobs"][0]["id"]})
after = state(att["id"])
version_rows = [row for row in after["variants"] if row["pipeline_version"] == after["pipeline_version"]]
if failed["error_code"] != "invalid_image" or info["jobs"][0]["attempt"] != 1:
raise Failure("permanent error was retried")
if replayed["result"].get("result") != "queued" or again["result"].get("result") != "already_ready":
raise Failure("replay result")
if len(version_rows) != 4 or restored["restored_sha256"] != corrupted["original_sha256"]:
raise Failure("replay did not keep one result")
return {
"corrupt_sha256": corrupted["original_sha256"],
"failed_code": failed["error_code"],
"attempt_at_fail": info["jobs"][0]["attempt"],
"restored_sha256": restored["restored_sha256"],
"replay": replayed["result"],
"second_replay": again["result"],
"variant_rows": len(version_rows),
"picture": picture,
}
def lease():
stop_worker()
alice, bob, _eve, chat = client.bootstrap()
arm("skip_inline_publish", None)
before = depth()
att, content = upload(alice, chat)
info = state(att["id"])
if info["unpublished"] != 1 or depth() != before:
raise Failure("lease setup published the job")
job_id = info["jobs"][0]["id"]
psql(
"UPDATE attachment_jobs SET status = 'processing', lease_owner = 'dead', "
"lease_until = now() - interval '2 minutes', updated_at = now() "
"WHERE id = '{id}'::uuid".format(id=job_id)
)
psql(
"UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = '{id}'::uuid".format(
id=att["id"]
)
)
psql("DELETE FROM outbox WHERE job_id = '{id}'::uuid AND published_at IS NULL".format(id=job_id))
started = time.monotonic()
start_worker()
ready = wait_status(bob, att["id"], "ready", 90)
elapsed = round(time.monotonic() - started, 3)
picture = check_picture(bob, ready, content)
info = state(att["id"])
if "lease_expired" not in info["events"] or info["jobs"][0]["attempt"] != 1:
raise Failure("expired lease was not resumed")
return {
"seconds_until_ready": elapsed,
"events": info["events"],
"attempt": info["jobs"][0]["attempt"],
"picture": picture,
}
def http_status(url):
req = urllib.request.Request(url, headers={"Accept": "application/json"})
try:
with urllib.request.urlopen(req, timeout=10) as response:
return response.status, response.read(4096).decode("utf-8", "replace")
except urllib.error.HTTPError as error:
return error.code, error.read(4096).decode("utf-8", "replace")
def leaked(text):
lowered = text.lower()
return [
word for word in (
"redis", "postgres", "minio", "password", "select", "rabbit", "amqp",
"secret", "database", "bucket", "messenger-attachments",
)
if word in lowered
]
def ready_503():
compose("stop", "redis")
time.sleep(2)
lb_status, lb_body = http_status(BASE + "/health/ready")
proc = run(
[
"docker", "exec", "mtusi-lab5-api-1", "python", "-c",
"import urllib.request,urllib.error\n"
"req=urllib.request.Request('http://127.0.0.1:8080/health/ready')\n"
"try:\n"
" r=urllib.request.urlopen(req, timeout=5)\n"
" print(r.status)\n"
" print(r.read(4096).decode())\n"
"except urllib.error.HTTPError as e:\n"
" print(e.code)\n"
" print(e.read(4096).decode())\n",
],
timeout=20,
)
lines = proc.stdout.decode("utf-8", "replace").splitlines()
api_status = int(lines[0])
api_body = "\n".join(lines[1:])
compose("start", "redis")
wait_health("mtusi-lab5-redis-1")
started = time.monotonic()
deadline = started + 40
recovered = None
while time.monotonic() < deadline:
try:
recovered, _body = http_status(BASE + "/health/ready")
except (urllib.error.URLError, TimeoutError, OSError):
recovered = 0
if recovered == 200:
break
time.sleep(0.5)
recovery_seconds = round(time.monotonic() - started, 3)
if lb_status != 503 or api_status != 503 or leaked(lb_body) or leaked(api_body):
raise Failure("ready 503 text")
if "Сервис временно недоступен" not in api_body or recovered != 200:
raise Failure("ready 503 text")
return {
"lb_status": lb_status,
"lb_body": lb_body,
"api_status": api_status,
"api_message": "Сервис временно недоступен",
"forbidden": [],
"recovered": recovered,
"recovery_seconds": recovery_seconds,
}
def main():
wait_health("mtusi-lab5-api-1")
wait_health("mtusi-lab5-api-2")
psql("UPDATE fault_injections SET armed = false WHERE armed")
start_worker()
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
if depth() == 0 and queues().get("attachments.retry", 0) == 0:
break
time.sleep(0.5)
steps = [
("durability", durability),
("outbox_gap", outbox_gap),
("duplicate", duplicate),
("reorder", reorder),
("stale_version", stale_version),
("crash_before_effect", lambda: crash("crash_before_effect")),
("crash_after_objects", lambda: crash("crash_after_objects")),
("crash_after_commit", lambda: crash("crash_after_commit")),
("transient_once", transient),
("permanent_replay", permanent_and_replay),
("lease", lease),
("ready_503", ready_503),
]
if len(sys.argv) > 1 and sys.argv[1] == "resume" and EVIDENCE.is_file():
done = [
item for item in json.loads(EVIDENCE.read_text(encoding="utf-8"))["results"] if item.get("ok")
]
results.extend(done)
steps = [item for item in steps if item[0] not in {row["name"] for row in done}]
failed = None
for name, fn in steps:
try:
scenario(name, fn)
except Exception as exc:
failed = exc
break
EVIDENCE.write_text(json.dumps({"results": results}, ensure_ascii=False, indent=2), encoding="utf-8")
if failed is not None:
print("FAIL " + type(failed).__name__ + ": " + str(failed), file=sys.stderr)
return 1
print("PASS pipeline")
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except Failure as exc:
print("FAIL " + str(exc), file=sys.stderr)
raise SystemExit(1)