diff --git a/.dockerignore b/.dockerignore index bc16e55..40bc41c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,9 @@ __pycache__ node_modules evidence .DS_Store +research +student-tests +scripts +docs +requirements-research.txt +*.md diff --git a/.env.example b/.env.example index 2b32655..8df1715 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,41 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Секреты (пароль БД, DATABASE_URL) создаёт scripts/init-secrets.sh в каталоге secrets/ — не в .env. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +# Срок жизни сессии; для проверки expiry запускайте с SESSION_TTL_SECONDS=2 (см. student-tests/test_expiry.sh) SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: +# TTL записи в кэше Valkey (не больше остатка жизни сессии) +SESSION_CACHE_TTL_SECONDS=60 +# Источник сессий: PostgreSQL (истина) + Valkey (кэш). Адрес кэша задан в compose.yaml (VALKEY_URL). SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +DB_POOL_MAX=10 +# Файлы секретов внутри контейнера (создаются scripts/init-secrets.sh) +DATABASE_URL_FILE=/run/secrets/database_url +# --- lab3 --- +# Число API-реплик (compose deploy.replicas); также можно: docker compose up -d --scale api=4 API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 +API_CPUS=1.0 +API_MEM=256m +POSTGRES_CPUS=1.0 +LB_CPUS=1.0 +# Пауза между SIGTERM и закрытием listener: LB успевает исключить реплику (rolling update без ошибок) +DRAIN_SECONDS=4 +# --- lab4 --- S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +S3_CPUS=1.0 +UPLOAD_CONCURRENCY=4 +# --- lab5 --- +RABBITMQ_CPUS=1.0 +WORKER_CPUS=1.0 +WORKER_MEM=768m +WORKER_REPLICAS=1 WORKER_CONCURRENCY=2 -JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +MAX_ATTEMPTS=5 +LEASE_SECONDS=60 +MAX_QUEUED_JOBS=200 +# --- lab6: наблюдаемость --- +OTEL_ENABLED=1 +# доля трасс при head-sampling в SDK (1.0 = все); профиль tail-sampling: COLLECTOR_CONFIG=/etc/otelcol/collector-tail.yaml +OTEL_TRACES_SAMPLER_ARG=1.0 +COLLECTOR_CONFIG=/etc/otelcol/collector.yaml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b759e5b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Keep shell hooks runnable after a Windows clone with core.autocrlf=true. +*.sh text eol=lf diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 0000000..a6996f2 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,25 @@ +name: Application smoke +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Prepare isolated lab credentials + run: | + sh scripts/init-secrets.sh + cp .env.example .env + docker compose --env-file .env config --quiet + - name: Build and wait for application + run: docker compose --env-file .env up -d --build --wait --wait-timeout 120 + - name: Check contract and public scenario + run: make check test PYTHON=python BASE_URL=http://127.0.0.1:8080 + - name: Stop disposable CI stack + if: always() + run: docker compose --env-file .env down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ee31f7a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1 +# Базовый образ закреплён тегом и digest (python 3.12.12, Debian bookworm). +ARG PYTHON_IMAGE=python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c + +# --- сборка зависимостей в отдельном слое (в runtime не попадают pip и кэш) --- +FROM ${PYTHON_IMAGE} AS builder +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_NO_CACHE_DIR=1 +WORKDIR /build +COPY requirements.txt . +RUN python -m venv /opt/venv && /opt/venv/bin/pip install -r requirements.txt + +# --- runtime --- +FROM ${PYTHON_IMAGE} +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/srv/src \ + APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab LIMIT_CONCURRENCY=1000 MIGRATIONS_DIR=/srv/migrations +RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home app +COPY --from=builder /opt/venv /opt/venv +WORKDIR /srv +COPY src/ /srv/src/ +COPY migrations/ /srv/migrations/ +# ревизия образа (меняется при rolling update; видна в журнале запуска) +ARG REV=dev +LABEL org.opencontainers.image.revision=$REV +ENV APP_REVISION=$REV +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-c", "import urllib.request,os,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT','8080'), timeout=2).status == 200 else 1)"] +# exec-форма: сервер = PID 1. Состояние хранится в PostgreSQL/Valkey, процесс stateless; +# масштабирование — репликами контейнера. SIGTERM: drain (readiness 503) -> graceful shutdown (см. app/serve.py). +CMD ["python", "-m", "app.serve"] diff --git a/Makefile b/Makefile index 5d86d17..765edd3 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,20 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# --- собственные цели студента --- +.PHONY: secrets up down student-test +secrets: + sh scripts/init-secrets.sh + +up: secrets + docker compose --env-file .env up -d --build + +down: + docker compose --env-file .env down + +student-test: + $(PYTHON) -m unittest discover -s student-tests -p "test_api.py" -v + $(PYTHON) -m unittest discover -s student-tests -p "test_attachments.py" -v + $(PYTHON) -m unittest discover -s student-tests -p "test_pipeline.py" -v + $(PYTHON) -m unittest discover -s student-tests -p "test_observability.py" -v diff --git a/REPORT.md b/REPORT.md index b97ae42..7c19be3 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,201 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Кушнаренко Николай Вячеславович, БПИ2302 +- Целевая оценка; выбранный трек 5, если нужен: **5**, трек 1 «Цена наблюдаемости» +- Commit/PR: ветка `solution/bpi2302-kushnarenko` в [своём форке](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko); текущие проверки от 12.11.2026 +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, OpenTelemetry Python SDK 1.45.0 (traces/metrics/logs, OTLP/HTTP), uvicorn, asyncpg, aiobotocore, aio-pika, Pillow; OpenTelemetry Collector contrib 0.161.0, Prometheus 3.15.0, Loki 3.7.8, Tempo 2.9.0, Grafana 13.2.3; RabbitMQ 4.1.8, SeaweedFS 4.48, PostgreSQL 17, Valkey 8, HAProxy 3.0. Код lab5 перенесён, бизнес-API не менялся. ИИ-ассистенты Claude/Codex использовались для помощи с кодом, тестами и оформлением; результаты ниже получены реальными запусками ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker Compose v2+ (выделить Docker не менее 7 ГиБ RAM; сумма лимитов контейнеров ≈6.1 ГиБ), bash (Git Bash на Windows), Python ≥ 3.9 для публичных тестов, Python ≥ 3.11 для исследований; на Windows `PYTHONUTF8=1`. +Версии всех образов закреплены тегом и digest в `compose.yaml`; зависимости — `requirements.txt` (включая `opentelemetry-*==1.45.0`). + +```sh +sh scripts/init-secrets.sh # secrets/*: БД, S3, RabbitMQ, пароль Grafana (secrets/grafana_admin_password) +cp .env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait --wait-timeout 180 +curl http://127.0.0.1:8080/health/ready # API +# Grafana: http://127.0.0.1:3000 пользователь admin, пароль — secrets/grafana_admin_password +python3 -m pip install -r research/requirements.txt -r requirements-dev.txt +python3 scripts/check_contract.py # make check +python3 scripts/validate_openapi.py # make validate +python3 tests/smoke.py --base-url http://127.0.0.1:8080 # make test (телеметрия включена) +python3 -m unittest discover -s student-tests -p test_observability.py -v # метрики, логи, трассы, связь API -> worker, отсутствие секретов +python3 student-tests/test_observability_retry.py # все попытки задания в одной трассе +mkdir -p evidence/my-run +python3 research/incidents.py --out evidence/my-run +python3 research/obs_cost.py --out evidence/my-run --profiles off on100 head10 tail --repeat 2 +python3 research/plot_obs_cost.py --out evidence/my-run +``` + +Windows: `scripts/restart.cmd`, `scripts/stop-one.cmd`. Остановка с сохранением данных — `docker compose --env-file .env stop`; **намеренное удаление учебных данных** — `docker compose --env-file .env down -v` (volumes `pgdata`, `valkeydata`, `seaweeddata`, `rabbitmqdata`, `promdata`, `lokidata`, `tempodata`, `grafanadata`). +Профиль без телеметрии: `OTEL_ENABLED=0`; head-sampling: `OTEL_TRACES_SAMPLER_ARG=0.1`; tail-sampling в Collector: `COLLECTOR_CONFIG=/etc/otelcol/collector-tail.yaml`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +API ×2 ─┐ ┌─▶ Prometheus (метрики, retention 7 д, volume promdata) + ├─OTLP/HTTP─▶ OTel Collector ─┼─▶ Tempo (трассы, retention 7 д, volume tempodata) +worker ─┘ (memory_limiter, batch, └─▶ Loki (логи, retention 7 д, volume lokidata) + scrub, tail_sampling*) ▲ +traceparent: API ─▶ outbox ─▶ RabbitMQ ─▶ worker Grafana (127.0.0.1:3000) читает все три источника +``` -## Доказательства критериев +| Роль | Сервис | Версия | Порты | Volume | Зачем | +| --- | --- | --- | --- | --- | --- | +| Collector | `otel-collector` | contrib 0.161.0 | OTLP 4317/4318, метрики 8888/8889 — **только сеть backend** | — | единая точка приёма, ограничение RAM, удаление чувствительных атрибутов, маршрутизация сигналов; это не хранилище | +| Метрики | `prometheus` | 3.15.0 | не публикуется | `promdata` | временные ряды, SLI/recording rules, alert rules, firing/resolved | +| Логи | `loki` | 3.7.8 | не публикуется | `lokidata` | структурные логи API и worker (OTLP), поиск по `trace_id` | +| Трассы | `tempo` | 2.9.0 | не публикуется | `tempodata` | распределённые трассы, TraceQL | +| UI | `grafana` | 13.2.3 | `127.0.0.1:3000` | `grafanadata` | datasources/dashboard из Git (provisioning), анонимный доступ выключен, пароль — secret | -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +**Что хранит что.** Grafana ничего не хранит из сигналов: это интерфейс запросов (PromQL, LogQL, TraceQL) и provisioning. Collector — маршрутизатор/обработчик (не БД). Метрики — Prometheus, логи — Loki, трассы — Tempo. Конфигурации (`infra/otel`, `infra/prometheus`, `infra/loki`, `infra/tempo`, `infra/grafana`) и dashboard JSON лежат в Git; провижининг datasources и dashboard автоматический. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### Сигналы и соответствие `contracts/observability.md` -## Эксперименты и отказы +| Сигнал контракта | Реализация (SDK) → имя в Prometheus / место | +| --- | --- | +| HTTP requests; HTTP duration (s) | `http.server.request.duration` (histogram, секунды, buckets 5 мс…10 с) → `http_server_request_duration_seconds_{bucket,sum,count}{http_request_method, http_route, http_response_status_code}`; число запросов = `_count` (отдельный счётчик не дублируется) | +| Jobs processed/failed; duration | `messenger.jobs.processed{job_type,outcome=ready|failed|retried|noop}` → `messenger_jobs_processed_total`; `messenger.job.duration` → `messenger_job_duration_seconds_*` | +| End-to-end time to ready | `messenger.job.time_to_ready` (от приёма до ready) → `messenger_job_time_to_ready_seconds_*` | +| Queue depth / oldest job age | `messenger.queue.depth`, `messenger.queue.oldest_job_age` → `messenger_queue_depth`, `messenger_queue_oldest_job_age_seconds` (считает API, т. е. метрика живёт и при остановленном worker) | +| DB pool used/waiting | `db.client.connections.usage{state}`, `db.client.connections.waiting` → `db_client_connections_usage`, `db_client_connections_waiting` | +| Worker busy/capacity | `messenger_worker_busy`, `messenger_worker_capacity` | +| CPU/RAM процессов | `process_memory_rss_bytes`, `process_cpu_time_seconds_total` (счётчик CPU-времени процесса) | +| Логи API и worker | структурный JSON в stdout **и** OTLP: `ts, level, service.name, msg(event), trace_id, span_id` (при активном спане), `request_id`; resource: `service.name` (`messenger-api`/`messenger-worker`), `service.version`, `service.instance.id`, `deployment.environment.name=lab` | +| Трассы API и worker | server-спан запроса (`HTTP POST /api/v1/…`), `db.connection`, `s3.put_object/get_object`, producer-спан `attachment.process publish`, consumer-спан `attachment.process process` — реальные, не статичные | +| Связь upload → publish → consume → S3/DB | W3C `traceparent/tracestate`: контекст запроса сохраняется в `outbox` (миграция 0006), relay кладёт его в поля сообщения (`jobs.schema.json`) и заголовки AMQP; worker продолжает ту же трассу (consumer — потомок producer; producer — потомок server-спана) | -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Низкая кардинальность.** `user_id/chat_id/message_id/attachment_id/request_id/filename` не используются как labels метрик (проверяет `test_no_high_cardinality_or_secret_labels`); маршрут — шаблон `/api/v1/chats/{chat_id}/messages`. В логах/спанах допустимы технические ID синтетических данных. **Секреты не экспортируются**: токены, пароли, `Authorization`, тела сообщений, изображения, SQL-текст, query-строки — не попадают ни в атрибуты, ни в логи; дополнительно Collector удаляет `http.request.header.authorization`, `db.statement`, `url.query` и т. п. (`attributes/scrub`). Проверка `test_secrets_and_message_bodies_are_not_exported`: уникальный текст сообщения, токен и пароль не находятся ни в Loki, ни в трассе, ни в метриках. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Health-проверки** LB (раз в секунду) не порождают спанов и access-логов (метрики остаются, исключены из SLI фильтром `http_route!~"/health.*"`); фоновые опросы БД (relay, sweeper) — без корневых спанов (`child_span`). -## Ограничения и следующие шаги +### Dashboard и SLI/SLO -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Dashboard `Messenger: API, очередь, worker` (`infra/grafana/dashboards/messenger.json`, генератор `build_dashboard.py`): RPS по маршрутам и кодам, **доля 5xx**, **p50/p95/p99** (`histogram_quantile` из `sum by (le)` гистограмм всех реплик — не среднее p95), задержки по маршрутам, DB pool (занято/простаивает/ожидают), задания по исходам, p50/p95 длительности обработки и **времени до ready**, глубина очереди, **возраст старейшей задачи**, saturation worker (busy/capacity), RSS/CPU, очередь и отказы экспорта Collector, логи ошибок (Loki), список алертов, последние трассы. Единицы и запросы описаны в подсказках панелей. + +- **SLI доступности**: доля бизнес-запросов (все маршруты кроме `/health`) без ответа 5xx. Числитель — 5xx, знаменатель — все запросы без `/health`; 4xx (неверный пароль, чужой чат) в отказ сервиса не входят. +- **SLO**: 99.5 % за окно 30 суток (бюджет ошибок 0.5 %); для демонстрации на локальном стенде окна алертов короткие (rate по 1 мин, `for` 15–20 с). +- Дополнительные SLI: p95 задержки (порог 0.5 с), доля 5xx **на маршруте загрузки** (малообъёмный, но критичный — общий SLI его деградации не заметит), p95 времени до ready. + +### Alert rules (`infra/prometheus/rules/messenger.yml`, runbook — `docs/runbooks.md`) + +| Алерт | Условие / окно | Для чего | +| --- | --- | --- | +| `HighHttpErrorRate` | 5xx > 5 % при ≥ 0.5 RPS, `for 20s`, rate[1m] | общий отказ сервиса | +| `HighUploadErrorRate` | 5xx на маршруте загрузки > 20 %, `for 15s` | отказ S3/очереди | +| `HighLatencyP95` | p95 > 0.5 с, `for 20s` | медленная БД/обработчик | +| `QueueLagGrowing` | старейшая задача > 30 с, `for 20s` | остановка/перегрузка worker | +| `WorkerSilent` | `absent(messenger_worker_capacity)`, `for 30s` | worker не шлёт метрики | +| `TelemetryExportFailing` | отказы экспорта Collector > 0, `for 15s` | сбой хранилищ телеметрии | + +Alerts вычисляет Prometheus; состояние (pending/firing/resolved) видно в Prometheus и на панели «Алерты» dashboard. Внешняя доставка не требуется и не настроена. + +### Collector: ограничение ресурсов и поведение при backpressure +`memory_limiter` (limit 280 MiB при лимите контейнера 384 MiB, spike 60 MiB; при приближении данные отклоняются, SDK отбрасывает пакет, приложение продолжает работать), `batch` (1024/2 с), экспортёры с **ограниченной очередью** (`sending_queue` 500) и повторами до 30 с, затем отбрасывание; SDK: очередь спанов/логов 2048, таймаут экспорта 3 с, **отказ Tempo/Loki не нарушил работу API** в воспроизведённом сценарии ниже. `metric_expiration: 20s` настроен в exporter. В текущем опыте остановки worker серия capacity оставалась видимой, поэтому диагноз основан на росте возраста очереди и нулевой скорости ready, а не на `WorkerSilent`. + +## Перенос предыдущей работы и воспроизводимость + +Перенесён собственный мессенджер из ЛР5: PostgreSQL — пользователи, чаты, сообщения, сессии, идемпотентность, metadata/outbox; Valkey — вспомогательный кэш; приватный SeaweedFS S3 — оригиналы и варианты; RabbitMQ — durable очередь; worker — асинхронная обработка изображений; HAProxy и две API-реплики. Загрузка по-прежнему возвращает 202, текстовое сообщение без вложений допустимо. Бизнес-контракт ЛР6 не меняется. + +| Компонент | CPU / RAM | Постоянные данные / готовность | +| --- | --- | --- | +| API ×2 | каждому 1 CPU / 256 MiB | stateless; HTTP `/health/ready` | +| worker | 1 CPU / 768 MiB | состояние в БД/S3/broker; heartbeat | +| PostgreSQL 17.11 | 1 CPU / 512 MiB | `pgdata`; `pg_isready`, миграции отдельным одноразовым сервисом | +| Valkey 8.1.10 | 0.5 CPU / 128 MiB | `valkeydata`; ping | +| SeaweedFS 4.48 | 1 CPU / 768 MiB | `seaweeddata`; bootstrap создаёт приватный bucket и проверяет ограниченные права API | +| RabbitMQ 4.1 (digest) | 1 CPU / 512 MiB | `rabbitmqdata`; diagnostics ping | +| HAProxy 3.0 (digest) | 1 CPU / 128 MiB | HTTP health; опубликован только `127.0.0.1:8080` | +| Collector / Prometheus / Loki / Tempo / Grafana | каждому 1 CPU; 384 / 512 / 512 / 1024 / 512 MiB | volumes и назначение выше; реальную доступность сигналов проверяют собственные тесты | + +Секреты и `.env` создаются локально, в Git не входят. `init-secrets.sh` идемпотентен; `.gitattributes` сохраняет LF у shell-скриптов при Windows clone. Внешние зависимости не нужно поднимать отдельно. Workflow `.github/workflows/application.yml` собирает приложение и запускает публичный сценарий; его наличие не означает успешного запуска CI на сервере Gitea. + +Выделенные исправления базы ЛР5: HAProxy не повторяет потенциально выполненный upload и сохраняет JSON-ответ приложения при отказе БД; reconciler защищает варианты, staging и незавершённые операции от удаления. Добавлены уже использованные в ЛР5 проверки TTL, зависимостей и reconciler; исправлено имя S3-сервиса в сценарии отказа. Версии и архитектура телеметрии сохранены. + +## Доказательства критериев (12.11.2026) + +| Проверка | Наблюдаемый результат | Доказательство | +| --- | --- | --- | +| Контракт, полная OpenAPI-валидация и public smoke при включённой телеметрии | PASS | [вывод](evidence/checks_2026-11-12.txt) | +| Собственные проверки | **66 уникальных тестов OK**: 12 API, 9 idempotency, 11 attachments, 6 topology, 9 pipeline, 5 observability, 1 retry trace, 2 reconcile, 7 resilience, 2 expiry, 2 dependencies | [команды](student-tests/README.md), [вывод](evidence/checks_2026-11-12.txt) | +| Реальные metrics/logs/traces, trace/span IDs, отсутствие секретов и высококардинальных labels | 5 observability-тестов OK; после исследования повторены | [вывод](evidence/recheck_2026-11-12/commands.txt) | +| Upload → producer → consumer → S3/DB; повторные попытки | 3 consumer-спана и 2 reschedule в одной трассе, parent-child проверен | [вывод](evidence/checks_2026-11-12.txt) | +| Перезапуск всего стенда | 13 контейнеров перезапущены, volumes сохранены, сессии и подтверждённые данные доступны | [lifecycle](evidence/recheck_2026-11-12/lifecycle.json), [повторная проверка](evidence/recheck_2026-11-12/recovery_checks.txt) | +| Отказ API-реплик по очереди | обе разные реплики остановлены; 120 и 119 успешных записей, 0 неуспешных попыток | [failover](evidence/recheck_2026-11-12/failover.json), [вывод](evidence/recheck_2026-11-12/recovery_checks.txt) | +| S3 bootstrap ×2 | повторный запуск успешен, anonymous 403, создание чужого bucket запрещено | [вывод](evidence/recheck_2026-11-12/recovery_checks.txt) | +| Grafana provisioning | 3 источника, dashboard `messenger-overview`, 22 панели, `provisioned=true` | [ответ API](evidence/recheck_2026-11-12/grafana_provisioning.json) | +| Чистый clone `6d87f54` | новые секреты и пустые volumes, build/up, public smoke и provisioning успешны; отдельный проект, порты 18080/13000 | [вывод](evidence/recheck_2026-11-12/clean_clone.txt), [override портов](evidence/recheck_2026-11-12/clean-ports.yaml) | +| Инциденты, alerts firing → resolved | все 4 воспроизведены | [JSON](evidence/recheck_2026-11-12/incidents.json), разбор ниже | +| Трек 1 | 4 профиля × 2 повтора | [исходные замеры](evidence/recheck_2026-11-12/obs_cost.json), [ресурсы](evidence/recheck_2026-11-12/research_runtime.json) | + +На Windows установлен Borland make, поэтому выполнены точные Python-команды целей `make check`, `make validate`, `make test` из Makefile, а не GNU make. Публичные `tests/`, `contracts/`, README, COURSE и CONTRIBUTING совпадают с исходным skeleton `470d9d6`. + +Первый persistence-запуск завершился таймаутом recovery hook после успешных 66 тестов. Полный обход S3 не успевал за 45 с на накопленных ~58 тыс. вложений даже при исправном backend. Hook теперь читает первый байт до 16 последних готовых оригиналов; это ограниченная проверка готовности, а не аудит всех исторических объектов. После исправления публичная проверка сохранности и оба failover пройдены. Исходный неуспешный вывод сохранён, повторный — отдельно в `recovery_checks.txt`. + +## Контролируемые инциденты + +Нагрузка: 4 цикла отправки сообщений с паузой 0.1 с и 10 циклов загрузки PNG с паузой 0.3 с. Реальная интенсивность зависит от задержки/отказа. `research/incidents.py` записывает метрики каждые ~3 с и локальные состояния Prometheus. Времена включают окно rate[1m] и `for` правила. + +| Воздействие | Алерт | До firing, с | От команды восстановления до resolved, с | +| --- | --- | ---: | ---: | +| Остановлен worker | `QueueLagGrowing` | 68.3 | 21.8 | +| DB: pg_sleep(0.8) | `HighLatencyP95` | 67.7 | 43.5 | +| Остановлен S3 | `HighUploadErrorRate` | 84.1 | 39.3 | +| Остановлены Tempo и Loki | `TelemetryExportFailing` | 69.1 | 67.5 | + +Диагностика по сохранённым наблюдениям (строки `diagnosis` в JSON — шаблоны скрипта; ниже учтены фактические поля, в том числе присутствующая capacity и нулевое ожидание DB pool): + +1. **Worker.** Возраст старейшего задания 60.7 с, скорость завершения ready 0.0/с; серия capacity присутствует: `True`. Сочетание роста возраста и отсутствия обработки указывает на worker. После восстановления алерт снят. +2. **Медленная БД.** p95 = 782.8 мс; ожидающих в DB pool: 0.0. Поиск обнаружил 20 трасс с `db.connection >300ms` и 20 медленных HTTP POST сообщений (лимит поиска 20). Задержка видна в спане БД; рост ожидания пула не требуется для этого диагноза. +3. **S3.** 5xx на upload = 100.0 %, общая доля = 1.1 %. Найдены 20 ошибочных трасс и 37 строк `s3_put_failed`. Локализация по маршруту, операциям S3 и логам отделяет этот отказ от общего отказа API. +4. **Tempo/Loki.** API: 62.6 RPS, 5xx = 0.0 %; очередь Collector = 32.0, send-failed spans ≈62.9/с, RSS = 239.0 MiB при лимите 384 MiB. Продолжающиеся бизнес-запросы и ошибки экспортёра показывают отказ пути телеметрии. Prometheus в этом сценарии доступен. Счётчик неуспешных экспортов включает повторы и не равен числу окончательно потерянных уникальных спанов; точное число потерянных логов/трасс этим опытом не установлено. + +![Метрики и переходы алертов](evidence/recheck_2026-11-12/incidents.png) + +## Трек 1: цена наблюдаемости + +**Гипотеза:** инструментирование добавляет CPU/RAM; head 10 % уменьшает ingest, но теряет большинство редких проблемных трасс. Tail policy сохраняет ошибки и задержки >500 мс, отправляя в Tempo лишь 10 % остальных трасс. + +**Методика:** 8 процессов-клиентов, 64 одновременных запроса; общие 40 синтетических пользователей и 40 чатов на всю серию, 90 % POST сообщений и 10 % GET. Каждый профиль: прогрев 5 с, замер 30 с, 25 с дозаписи, **2 повтора**; перед профилем пересозданы API/worker/Collector. Во всех профилях одинаковая инъекция: 2 % запросов → 500, 2 % → +0.6 с. RPS, ошибки и latency в таблице относятся к POST-сообщениям; успешные RPS исключают внедрённые ошибки. Для off отключены сигналы API/worker, но контейнеры хранилищ остаются запущены. + +Машина: i9-12900KS (16 ядер / 24 потока), 95.8 ГиБ RAM, Windows/Docker Desktop (WSL2); Docker доступны 24 CPU и 46.9 ГиБ RAM. Клиент Python 3.11.9, приложение Python 3.12.12. Лимиты контейнеров одинаковы во всех профилях (таблица выше); другой Docker-проект оставался в фоне. Накопленные данные не очищались. Размеры БД после серии: {'users': 739, 'chats': 425, 'messages': 1610549, 'attachments': 68898}; отдельный счётчик сообщений до серии не снимался. [Размер набора](evidence/recheck_2026-11-12/dataset.json), [хэши исходников и среда](evidence/recheck_2026-11-12/source_metadata.json). + +Среднее двух прогонов; RAM — максимальный пик из двух. CPU API — **на одну реплику**, 100 % соответствует одному ядру. + +| Профиль | Успешные RPS (min–max) | Ошибки, % | p50 / p95 / p99, мс | CPU API, % | RAM API, MiB | CPU / RAM Collector | +| --- | ---: | ---: | --- | ---: | ---: | --- | +| Без телеметрии | 1368 (1348–1388) | 2.05 | 16.7 / 73.0 / 618.1 | 50.2 | 76.4 | 0.1 % / 45.1 MiB | +| Head 100 % | 1245 (1239–1251) | 2.01 | 24.6 / 100.3 / 629.7 | 79.5 | 83.2 | 3.3 % / 75.0 MiB | +| Head 10 % | 1220 (1137–1303) | 1.98 | 23.9 / 101.5 / 626.8 | 68.5 | 83.7 | 1.9 % / 68.9 MiB | +| SDK 100 % + tail | 1291 (1282–1300) | 2.07 | 23.5 / 90.2 / 628.6 | 77.4 | 75.5 | 4.0 % / 108.4 MiB | + +Объём — среднее за измерительное окно с дозаписью; logs и metric points считаются отдельно от spans. + +| Профиль | Принято spans | Отправлено spans | Logs | Metric points | Найдено error / внедрено 500 | Найдено slow / медленных у клиента | +| --- | ---: | ---: | ---: | ---: | --- | --- | +| Без телеметрии | 0 | 0 | 0 | 0 | 0 / 873 | 0 / 856 | +| Head 100 % | 89886 | 89886 | 44324 | 396 | 864 / 780 | 876 / 772 | +| Head 10 % | 8898 | 8982 | 43176 | 396 | 85 / 750 | 86 / 736 | +| SDK 100 % + tail | 91882 | 13138 | 45226 | 396 | 916 / 828 | 858 / 768 | + +**Результат.** Средний CPU API: off 50.2 %, on100 79.5 %, head10 68.5 %, tail 77.4 %. Изменение on100 относительно off: 29.3 п. п. Head sampling сокращает число спанов, но логи и метрики остаются; поэтому снижение CPU не обязано быть пропорционально sampling rate. При tail SDK продолжает формировать 100 % спанов, а Collector отбирает их перед Tempo: отправлено на 85.7 % меньше, чем принято. Это экономия ingest спанов, не прямое измерение дискового объёма. + +**Потери и выбор конфигурации.** Head 10 % сохраняет около 11.3 % ошибочных и 11.8 % медленных трасс по приближённому сопоставлению окон. Tail сохраняет значительно больше проблемных трасс при меньшем ingest, поэтому выбран как проверенная сбалансированная конфигурация: SDK 100 %, `collector-tail.yaml`, все ERROR, все >500 мс, 10 % остальных. Цена — CPU SDK и память Collector для ожидания решения. Для демонстрации/тестов по умолчанию восстановлен on100 без инъекции отказов. + +Сумма `receiver_refused_spans` / `exporter_send_failed_spans` по двум повторам: off 0/0, on100 0/0, head10 0/0, tail 0/0. Tempo `discarded_spans_total` до/после серии: null / null. `null` означает отсутствие ряда, а не измеренный ноль. Эти счётчики описывают наблюдавшиеся отказы приёма/экспорта; сами по себе не доказывают отсутствие любых потерь на SDK. + +![Сравнение профилей](evidence/recheck_2026-11-12/obs_cost.png) + +На графике RPS усреднены по двум прогонам, CPU/ingest/отношения взяты из второго; отношения визуально ограничены 100 %. В таблицах выше приведены средние и исходные количества, подробные результаты каждого повтора — в JSON. + +**Ограничения методики:** два коротких повтора на одном ПК с фоновыми задачами, без доверительных интервалов; различия RPS могут быть обычным разбросом. Данные накапливаются, профили идут в фиксированном порядке. Tempo-поиск включает 5 с прогрева, поэтому числитель иногда больше знаменателя: это приближённое сопоставление, не точная вероятность сохранения или доказательство 100 % сохранности. Поиск ограничен 5000 трассами (для всех HTTP лимит достижим); точное число потерянных уникальных спанов SDK не измерено. Размер volumes не используется для вывода: compaction и flush искажают короткие замеры. + +## Ограничения решения + +- Один Docker-host; БД, LB, broker, worker и каждое хранилище телеметрии — по одному экземпляру. Отказ host/volume не покрывается. +- Проверена остановка Tempo/Loki; отдельная остановка Collector/Prometheus в текущем исследовании не выполнялась. Ограниченные очереди допускают потери телеметрии при длительном простое. +- Recovery hook проверяет недавние готовые оригиналы; публичная persistence-проверка отдельно проверяет созданные ею данные. Это не проверка каждого исторического объекта S3. +- SLO 99.5 % за 30 суток задан как цель; короткие эксперименты не доказывают выполнение месячного SLO. Алерты локальные, без внешней доставки. +- Исторические `evidence/incidents.json`, `evidence/obs_cost.json` и графики от октября оставлены как архив; все выводы текущей сдачи опираются на `evidence/recheck_2026-11-12/`. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..0b9f09d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,337 @@ +# Стенд lab6: HAProxy -> API x N -> PostgreSQL (метаданные) + SeaweedFS (приватный S3, байты) + Valkey (кэш сессий). +# Секреты: scripts/init-secrets.sh создаёт secrets/* (в Git не попадают). Запуск и проверки — см. REPORT.md. +name: mtusi-lab6 + +x-common: &common + restart: unless-stopped + logging: + driver: json-file + options: {max-size: "10m", max-file: "3"} + +services: + postgres: + <<: *common + image: postgres:17.11-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: + - pgdata:/var/lib/postgresql/data + command: ["postgres", "-c", "max_connections=200", "-c", "shared_buffers=128MB", "-c", "log_min_duration_statement=500"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + mem_limit: 512m + cpus: ${POSTGRES_CPUS:-1.0} + networks: [backend] + + valkey: + <<: *common + image: valkey/valkey:8.1.10-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", + "--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl", "--save", ""] + volumes: + - valkeydata:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 12 + mem_limit: 128m + cpus: 0.5 + networks: [backend] + + # Приватное S3-совместимое хранилище SeaweedFS (all-in-one: master + volume + filer + S3-шлюз). + # Порты наружу не публикуются: доступ только из сети backend и только по ключам из s3.json (identities). + # Анонимный доступ запрещён (при заданных identities запрос без подписи получает 403). + seaweedfs: + <<: *common + image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d + entrypoint: ["/bin/sh", "-c"] + command: ["exec weed server -dir=/data -ip=seaweedfs -master.volumeSizeLimitMB=512 -volume.max=30 -filer -s3 -s3.port=8333 -s3.config=/run/secrets/seaweed_s3_config"] + secrets: [seaweed_s3_config] + volumes: + - seaweeddata:/data + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status && nc -z 127.0.0.1 8333"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 10s + mem_limit: 768m + cpus: ${S3_CPUS:-1.0} + networks: [backend] + + # Идемпотентный bootstrap: bucket создаётся, если его нет; проверяется, что анонимный доступ запрещён и что ключ API работает. + s3-init: + build: . + image: mtusi-messenger:lab6 + command: ["python", "-m", "app.s3_bootstrap"] + environment: + S3_ENDPOINT: http://seaweedfs:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/s3_admin_access_key + S3_ADMIN_SECRET_KEY_FILE: /run/secrets/s3_admin_secret_key + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + secrets: [s3_admin_access_key, s3_admin_secret_key, s3_access_key, s3_secret_key] + depends_on: + seaweedfs: {condition: service_healthy} + restart: "no" + networks: [backend] + + # Брокер: durable quorum-очередь, persistent-сообщения, publisher confirms; данные на volume rabbitmqdata. + # hostname фиксирован: имя узла RabbitMQ определяет каталог данных, иначе очередь не переживёт пересоздание контейнера. + rabbitmq: + <<: *common + image: rabbitmq:4.1-alpine@sha256:6a0f7984185cd2cdbf78a184b4c17abe88ecbbda6006a7acf17dbc7e4307f4e2 + hostname: rabbitmq + # RabbitMQ 4.x: RABBITMQ_DEFAULT_USER/PASS объявлены устаревшими -> учётные данные в конфигурационном файле (secret) + secrets: + - source: rabbitmq_conf + target: /etc/rabbitmq/conf.d/20-credentials.conf + mode: 0444 + volumes: + - rabbitmqdata:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 8s + retries: 12 + mem_limit: 512m + cpus: ${RABBITMQ_CPUS:-1.0} + networks: [backend] + + migrate: + build: . + image: mtusi-messenger:lab6 + command: ["python", "-m", "app.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: {condition: service_healthy} + restart: "no" + networks: [backend] + + api: + <<: *common + build: . + image: mtusi-messenger:lab6 + environment: + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL_FILE: /run/secrets/database_url + VALKEY_URL: redis://valkey:6379/0 + S3_ENDPOINT: http://seaweedfs:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DRAIN_SECONDS: ${DRAIN_SECONDS:-4} + MAX_INFLIGHT: ${MAX_INFLIGHT:-64} + ACCESS_LOG: ${ACCESS_LOG:-1} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4} + FAULT_INJECTION: ${API_FAULT_INJECTION:-} + BROKER_URL_FILE: /run/secrets/broker_url + MAX_QUEUED_JOBS: ${MAX_QUEUED_JOBS:-200} + SERVICE_NAME: messenger-api + OTEL_ENABLED: ${OTEL_ENABLED:-1} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + OTEL_QUEUE_SIZE: ${OTEL_QUEUE_SIZE:-2048} + OTEL_BATCH_SIZE: ${OTEL_BATCH_SIZE:-512} + secrets: [database_url, s3_access_key, s3_secret_key, broker_url] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + valkey: {condition: service_healthy} + stop_grace_period: 15s + mem_limit: ${API_MEM:-256m} + cpus: ${API_CPUS:-1.0} + deploy: + replicas: ${API_REPLICAS:-2} + networks: [backend] + + # Worker: отдельный процесс (тот же образ). Берёт задания из RabbitMQ, обрабатывает изображения, пишет результат в S3 и БД. + worker: + <<: *common + build: . + image: mtusi-messenger:lab6 + command: ["python", "-m", "app.worker"] + environment: + SERVICE_NAME: messenger-worker + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://seaweedfs:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + BROKER_URL_FILE: /run/secrets/broker_url + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + MAX_ATTEMPTS: ${MAX_ATTEMPTS:-5} + LEASE_SECONDS: ${LEASE_SECONDS:-60} + RETRY_BASE_DELAY: ${RETRY_BASE_DELAY:-2} + FAULT_INJECTION: ${WORKER_FAULT_INJECTION:-} + OTEL_ENABLED: ${OTEL_ENABLED:-1} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + OTEL_QUEUE_SIZE: ${OTEL_QUEUE_SIZE:-2048} + OTEL_BATCH_SIZE: ${OTEL_BATCH_SIZE:-512} + ACCESS_LOG: "0" + secrets: [database_url, s3_access_key, s3_secret_key, broker_url] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + rabbitmq: {condition: service_healthy} + healthcheck: + test: ["CMD", "python", "-c", "import os,time,sys; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker.alive') < 30 else 1)"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 20s + stop_grace_period: 30s + mem_limit: ${WORKER_MEM:-768m} + cpus: ${WORKER_CPUS:-1.0} + deploy: + replicas: ${WORKER_REPLICAS:-1} + networks: [backend] + + # ------------------------------ наблюдаемость (lab6) ------------------------------ + # OTLP-порты 4317/4318 и метрики Collector 8888/8889 доступны только внутри сети backend (наружу не публикуются). + otel-collector: + <<: *common + image: otel/opentelemetry-collector-contrib:0.161.0@sha256:fd328de2552466ad78385e1b1289c3f2402b1c45f265b252aab1955b42845ac1 + command: ["--config=${COLLECTOR_CONFIG:-/etc/otelcol/collector.yaml}"] + volumes: + - ./infra/otel:/etc/otelcol:ro + mem_limit: 384m + cpus: ${COLLECTOR_CPUS:-1.0} + networks: [backend] + + prometheus: + <<: *common + image: prom/prometheus:v3.15.0@sha256:efd719c99d83b060d9daefdcf00360461adf279f45ef5391f8d111892118753e + command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d", + "--web.enable-lifecycle"] + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - promdata:/prometheus + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"] + interval: 10s + timeout: 3s + retries: 6 + mem_limit: 512m + cpus: 1.0 + networks: [backend] + + loki: + <<: *common + image: grafana/loki:3.7.8@sha256:1107dd5274e0ada47e42472b7a7e71f3b2a2fe878878108f3e2f9e51528f0193 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - lokidata:/loki + mem_limit: 512m + cpus: 1.0 + networks: [backend] + + tempo: + <<: *common + image: grafana/tempo:2.9.0@sha256:65a5789759435f1ef696f1953258b9bbdb18eb571d5ce711ff812d2e128288a4 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempodata:/var/tempo + mem_limit: 1g + cpus: 1.0 + networks: [backend] + + # Grafana: единственный опубликованный сервис наблюдаемости, только 127.0.0.1:3000. Datasources, dashboard и alert-вью + # подключаются provisioning-ом из Git (infra/grafana). Анонимный доступ выключен; пароль admin — secret. + grafana: + <<: *common + image: grafana/grafana:13.2.3@sha256:b28bae15e219c998fb0e0424ed724930cc61b1f61fb404d47c862f9a23f9e572 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json + secrets: [grafana_admin_password] + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + depends_on: [prometheus, loki, tempo] + ports: + - "127.0.0.1:3000:3000" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health"] + interval: 10s + timeout: 3s + retries: 12 + mem_limit: 512m + cpus: 1.0 + networks: [backend] + + lb: + <<: *common + image: haproxy:3.0-alpine@sha256:56b887da77428b7a6621e59e480cdbd330cc805c22d3cedb66ceea76ffdea2c6 + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + - api + ports: + - "127.0.0.1:8080:8080" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/healthz"] + interval: 5s + timeout: 3s + retries: 6 + mem_limit: 128m + cpus: ${LB_CPUS:-1.0} + stop_grace_period: 10s + networks: [backend] + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + seaweed_s3_config: + file: ./secrets/seaweed_s3.json + s3_admin_access_key: + file: ./secrets/s3_admin_access_key + s3_admin_secret_key: + file: ./secrets/s3_admin_secret_key + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key + rabbitmq_conf: + file: ./secrets/rabbitmq.conf + grafana_admin_password: + file: ./secrets/grafana_admin_password + broker_url: + file: ./secrets/broker_url + +volumes: + pgdata: + valkeydata: + seaweeddata: + rabbitmqdata: + promdata: + lokidata: + tempodata: + grafanadata: + +networks: + backend: diff --git a/docs/pull_request_lab6.md b/docs/pull_request_lab6.md new file mode 100644 index 0000000..71312fa --- /dev/null +++ b/docs/pull_request_lab6.md @@ -0,0 +1,41 @@ +## Лабораторная 6 + +Автор / группа: Кушнаренко Николай Вячеславович, БПИ2302 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: Трек 1 — цена наблюдаемости + +Что реализовано и что перенесено: +Собственный мессенджер из ЛР5: две API-реплики, HAProxy, PostgreSQL, Valkey, приватный S3, RabbitMQ и worker. Бизнес-API сохранён. Добавлены OpenTelemetry SDK/Collector, Prometheus, Loki, Tempo и Grafana; provisioning трёх источников и dashboard из 22 панелей, гистограммы, SLI/SLO и шесть alert rules. Контекст W3C проходит API → outbox → broker → worker → S3/DB, включая retry. Есть memory limiter, batch, ограниченные очереди, удаление чувствительных атрибутов. Исправления базы ЛР5 (HAProxy/reconciler), Windows LF и ограниченный S3 recovery hook выделены в REPORT.md. + +Команды запуска из чистого clone: +```sh +sh scripts/init-secrets.sh +cp .env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait --wait-timeout 180 +python3 scripts/check_contract.py +python3 tests/smoke.py --base-url http://127.0.0.1:8080 +``` +Полные команды своих тестов и исследований — [student-tests/README.md](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko/student-tests/README.md). + +Результаты проверок и ссылка на REPORT.md: +[REPORT.md](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko/REPORT.md), [Word](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko/docs/report_lab6.docx), [вывод 66 тестов](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko/evidence/checks_2026-11-12.txt), [текущие доказательства](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko/evidence/recheck_2026-11-12). +- контракт, полная OpenAPI-валидация, публичный smoke — PASS; 66 собственных тестов — OK; +- отдельный чистый clone: build/up на новых секретах и пустых volumes, public smoke и provisioning Grafana — PASS; +- перезапуск 13 контейнеров сохранил volumes, сессии и данные; обе API-реплики проверены отдельными failover; +- четыре инцидента (worker, медленная БД, S3, Tempo/Loki): alerts firing → resolved; отказ Tempo/Loki не нарушил бизнес-запросы; +- трек 1: off/on100/head10/tail, одинаковая нагрузка, два повтора; latency, CPU/RAM, ingest и сохранение редких трасс измерены. Tail уменьшил отправку спанов относительно приёма на 85.7 %; +- на Windows выполнены Python-эквиваленты целей make check/validate/test. Серверный CI Gitea не заявляется как пройденный. + +Ограничения / известные ошибки: +Один Docker-host и по одному экземпляру stateful-компонентов; возможны потери телеметрии при простое backend. Два коротких повтора, фоновые процессы и накопленная БД ограничивают точность сравнения. TraceQL-окно включает прогрев, поэтому отношения сохранённых трасс приближённые. Первый полный S3 recovery probe превысил таймаут на ~58 тыс. вложений; после ограничения до 16 последних оригиналов persistence/failover прошли, оба вывода сохранены. + +Источники и использование ИИ: +Собственный код ЛР5, Python/uvicorn, asyncpg, aiobotocore, aio-pika, Pillow, OpenTelemetry и указанные хранилища. Первичные материалы — ссылки README; версии закреплены в репозитории. Claude/Codex помогали с кодом, тестами и оформлением; результаты проверены реальными запусками. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены эквивалентными Python-командами; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/docs/report_lab6.docx b/docs/report_lab6.docx new file mode 100644 index 0000000..47610b5 Binary files /dev/null and b/docs/report_lab6.docx differ diff --git a/docs/runbooks.md b/docs/runbooks.md new file mode 100644 index 0000000..99cefb6 --- /dev/null +++ b/docs/runbooks.md @@ -0,0 +1,34 @@ +# Runbooks (локальный стенд lab6) + +Grafana — `http://127.0.0.1:3000` (пользователь `admin`, пароль — `secrets/grafana_admin_password`), dashboard «Messenger: API, очередь, worker». +Prometheus/Loki/Tempo доступны только внутри сети стенда: запросы — через Grafana (Explore) либо `docker compose exec prometheus wget -qO- http://loki:3100/...`. + +Общий порядок диагностики: **метрика → трасса → лог**. Метрика показывает *что* и *когда* (алерт, dashboard), трасса — *где* потрачено время/где ошибка +(по `trace_id` из лога или из Tempo), лог — *почему* (событие и безопасные поля без тел сообщений и токенов). +Связка лог → трасса: в Loki у каждой записи есть `trace_id`/`span_id` (в Grafana — ссылка «Tempo»); трасса → логи: кнопка «Logs for this span». + +## HighErrorRate +Алерт `HighHttpErrorRate`: доля ответов 5xx среди бизнес-запросов (без `/health`) > 5 % при ≥ 0.5 запроса/с, 20 с. 4xx (неверный пароль, 404) в SLI не входят. +1. Dashboard → «RPS по статус-кодам» и «Доля ошибок»: какой маршрут даёт 5xx (`http_route` в `sum by (http_route)`). +2. Tempo: TraceQL `{ status = error && resource.service.name = "messenger-api" }` — в трассе виден спан с ошибкой (`s3.put_object`, `db.connection`, …) и `http.response.status_code`. +3. Loki: `{service_name="messenger-api"} | status >= 500` или поиск событий `s3_put_failed`, `db_unavailable`, `unhandled_exception` (поле `error_type`). +4. Действия: S3 недоступен → `docker compose ps seaweedfs`, `docker compose start seaweedfs`; БД → `docker compose ps postgres`, readiness реплик; затем алерт возвращается в resolved через ~1 мин (окно `rate[1m]`). + +## HighLatency +Алерт `HighLatencyP95`: p95 всех бизнес-запросов (из суммарной гистограммы всех реплик) > 0.5 с. +1. Dashboard «Latency p50/p95/p99» и «Latency p95 по маршрутам» → какой маршрут. +2. Отличить **медленную БД** от **медленного обработчика**: «DB pool: занято/ожидают» (рост `waiting`) и длительность спанов `db.connection` в медленной трассе (`{ name = "db.connection" && duration > 300ms }`). Если почти всё время запроса внутри `db.connection` — БД; если время в самом серверном спане вне дочерних — код приложения/CPU (`process_cpu_time`). +3. БД: `log_min_duration_statement=500` пишет медленные запросы в лог `postgres`; проверить блокировки/триггеры (`pg_stat_activity`). + +## QueueLag +Алерт `QueueLagGrowing`: возраст старейшего задания (`queued`/`processing`) > 30 с, 20 с. +1. Dashboard «Возраст старейшей задачи», «Глубина очереди», «Worker: занято/ёмкость», «Задания по исходам». +2. Нет метрик worker / `busy=0` и растёт возраст → worker остановлен или не подключён к брокеру: `docker compose ps worker rabbitmq`, `docker compose logs worker`. `busy == capacity` при росте очереди → не хватает мощности (увеличить `WORKER_CONCURRENCY`/реплики, см. отчёт lab5). +3. Много `retried`/`failed` → причина в логах worker (`job_rescheduled`, `reason`) и в трассе consumer-спана (S3/БД). Реестр: `docker compose run --rm -T worker python -m app.replay list`. +4. После устранения причины очередь рассасывается, алерт переходит в resolved. + +## Telemetry +Алерт `TelemetryExportFailing`: Collector не может отправить данные в Tempo/Loki. +1. Это сбой наблюдаемости, а не приложения: бизнес-метрики API (ошибки, RPS) не меняются, приложение работает (SDK и Collector ограничивают очередь/RAM и отбрасывают данные). +2. Проверить: `docker compose ps tempo loki`; метрики Collector `otelcol_exporter_queue_size`, `otelcol_exporter_send_failed_*`, `otelcol_process_memory_rss`. +3. Запустить хранилища: `docker compose start tempo loki`. Часть трасс/логов может потеряться после исчерпания очереди или срока повторов. `send_failed` считает неуспешные попытки, включая повторы, и не даёт точного числа потерянных уникальных спанов. При остановке только Tempo/Loki метрики продолжают идти по отдельному пути Collector → Prometheus. diff --git a/evidence/checks.txt b/evidence/checks.txt new file mode 100644 index 0000000..8732146 --- /dev/null +++ b/evidence/checks.txt @@ -0,0 +1,66 @@ +$ python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +$ python3 tests/smoke.py (телеметрия включена) +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +# контейнеры до перезапуска: +/mtusi-lab6-api-2 StartedAt=2026-10-03T02:04:24.831646035Z +/mtusi-lab6-api-1 StartedAt=2026-10-03T02:04:24.660308554Z +/mtusi-lab6-worker-1 StartedAt=2026-10-03T02:04:23.350829139Z +/mtusi-lab6-rabbitmq-1 StartedAt=2026-10-03T02:04:19.141968204Z +/mtusi-lab6-seaweedfs-1 StartedAt=2026-10-03T02:04:19.613928237Z +/mtusi-lab6-postgres-1 StartedAt=2026-10-03T02:04:18.665413808Z +/mtusi-lab6-valkey-1 StartedAt=2026-10-03T02:04:18.435301589Z +/mtusi-lab6-otel-collector-1 StartedAt=2026-10-03T02:04:19.399662403Z +/mtusi-lab6-prometheus-1 StartedAt=2026-10-03T02:04:16.212964886Z +/mtusi-lab6-loki-1 StartedAt=2026-10-03T02:04:19.815321273Z +/mtusi-lab6-tempo-1 StartedAt=2026-10-03T02:04:18.87934529Z +/mtusi-lab6-grafana-1 StartedAt=2026-10-03T02:04:18.083530623Z +$ python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh (Windows: restart.cmd; перезапускаются API, worker, брокер, S3, БД, кэш и стек наблюдаемости) +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +# контейнеры после перезапуска: +/mtusi-lab6-api-2 StartedAt=2026-10-03T02:05:18.916843054Z +/mtusi-lab6-api-1 StartedAt=2026-10-03T02:05:20.476426854Z +/mtusi-lab6-worker-1 StartedAt=2026-10-03T02:05:19.075989142Z +/mtusi-lab6-rabbitmq-1 StartedAt=2026-10-03T02:05:15.668495976Z +/mtusi-lab6-seaweedfs-1 StartedAt=2026-10-03T02:05:15.807584366Z +/mtusi-lab6-postgres-1 StartedAt=2026-10-03T02:05:14.948208653Z +/mtusi-lab6-valkey-1 StartedAt=2026-10-03T02:05:14.108577454Z +/mtusi-lab6-otel-collector-1 StartedAt=2026-10-03T02:05:13.357193004Z +/mtusi-lab6-prometheus-1 StartedAt=2026-10-03T02:05:14.678083736Z +/mtusi-lab6-loki-1 StartedAt=2026-10-03T02:05:15.497739002Z +/mtusi-lab6-tempo-1 StartedAt=2026-10-03T02:05:15.099095895Z +/mtusi-lab6-grafana-1 StartedAt=2026-10-03T02:05:15.320432594Z +$ failover (1-я реплика) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 0, "first_recovery_seconds": 2.36} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +$ failover (2-я реплика) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 0, "first_recovery_seconds": 2.375} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +$ python -m unittest test_api.py +Ran 12 tests in 2.950s +OK +$ python -m unittest test_idempotency.py +Ran 9 tests in 2.070s +OK +$ python -m unittest test_attachments.py +Ran 11 tests in 6.104s +OK +$ python student-tests/test_pipeline.py (Pillow) +Ran 9 tests in 11.272s +OK +$ python student-tests/test_topology.py +Ran 6 tests in 2.857s +OK +$ python student-tests/test_observability.py (метрики, логи, трассы, связь API->worker, нет секретов) +Ran 5 tests in 7.658s +OK +$ python student-tests/test_observability_retry.py (все попытки задания — одна трасса) +Ran 1 test in 18.312s +OK +$ python student-tests/test_resilience5.py (окна отказа конвейера, телеметрия включена) +Ran 7 tests in 139.292s +OK +CHECKS-DONE diff --git a/evidence/checks_2026-11-12.txt b/evidence/checks_2026-11-12.txt new file mode 100644 index 0000000..01fcf1b --- /dev/null +++ b/evidence/checks_2026-11-12.txt @@ -0,0 +1,182 @@ +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +exit=0 + +$ python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +exit=0 + +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +exit=0 + +$ python -m unittest discover -s student-tests -p test_api.py -v +test_bearer_rules (test_api.AuthTest.test_bearer_rules) ... ok +test_concurrent_registration_of_same_username (test_api.AuthTest.test_concurrent_registration_of_same_username) ... ok +test_logout_revokes_in_database_and_cache (test_api.AuthTest.test_logout_revokes_in_database_and_cache) ... ok +test_password_is_not_exposed_or_stored_plain (test_api.AuthTest.test_password_is_not_exposed_or_stored_plain) ... ok +test_register_validation (test_api.AuthTest.test_register_validation) ... ok +test_session_shape_entropy_and_logout (test_api.AuthTest.test_session_shape_entropy_and_logout) ... ok +test_token_only_stored_as_hash (test_api.AuthTest.test_token_only_stored_as_hash) ... ok +test_wrong_password_and_unknown_user_are_indistinguishable (test_api.AuthTest.test_wrong_password_and_unknown_user_are_indistinguishable) ... ok +test_chat_creation_is_atomic (test_api.DataTest.test_chat_creation_is_atomic) ... ok +test_foreign_chat_empty_message_bad_json (test_api.DataTest.test_foreign_chat_empty_message_bad_json) ... ok +test_multi_page_history_and_cursor_binding (test_api.DataTest.test_multi_page_history_and_cursor_binding) ... ok +test_parallel_writes_from_many_clients (test_api.DataTest.test_parallel_writes_from_many_clients) ... ok + +---------------------------------------------------------------------- +Ran 12 tests in 2.581s + +OK +exit=0 + +$ python -m unittest discover -s student-tests -p test_idempotency.py -v +test_concurrent_same_key_creates_exactly_one_message_across_replicas (test_idempotency.IdempotencyTest.test_concurrent_same_key_creates_exactly_one_message_across_replicas) ... ok +test_different_body_same_key_is_409 (test_idempotency.IdempotencyTest.test_different_body_same_key_is_409) ... ok +test_expired_key_is_reusable_after_ttl (test_idempotency.IdempotencyTest.test_expired_key_is_reusable_after_ttl) ... ok +test_key_validation (test_idempotency.IdempotencyTest.test_key_validation) ... ok +test_replay_rechecks_session_and_membership (test_idempotency.IdempotencyTest.test_replay_rechecks_session_and_membership) ... ok +test_replay_returns_same_message_and_creates_no_duplicate (test_idempotency.IdempotencyTest.test_replay_returns_same_message_and_creates_no_duplicate) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_scope_is_user_and_chat) ... ok +test_semantically_equal_body_is_replayed (test_idempotency.IdempotencyTest.test_semantically_equal_body_is_replayed) ... ok +test_session_is_shared_between_replicas_without_relogin (test_idempotency.IdempotencyTest.test_session_is_shared_between_replicas_without_relogin) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 2.087s + +OK +exit=0 + +$ python -m unittest discover -s student-tests -p test_attachments.py -v +test_aborted_upload_leaves_nothing (test_attachments.AttachmentTest.test_aborted_upload_leaves_nothing) ... ok +test_bucket_is_private_outside_the_api (test_attachments.AttachmentTest.test_bucket_is_private_outside_the_api) ... ok +test_corrupted_and_spoofed_files_are_415_not_500 (test_attachments.AttachmentTest.test_corrupted_and_spoofed_files_are_415_not_500) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTest.test_dangerous_filenames_are_400) ... ok +test_decompression_bomb_header_is_413 (test_attachments.AttachmentTest.test_decompression_bomb_header_is_413) ... ok +test_empty_body_is_400 (test_attachments.AttachmentTest.test_empty_body_is_400) ... ok +test_foreign_user_cannot_upload_read_or_download (test_attachments.AttachmentTest.test_foreign_user_cannot_upload_read_or_download) ... ok +test_idempotent_replay_with_attachments (test_attachments.AttachmentTest.test_idempotent_replay_with_attachments) ... ok +test_message_with_attachments_and_chat_binding (test_attachments.AttachmentTest.test_message_with_attachments_and_chat_binding) ... ok +test_png_and_jpeg_roundtrip_with_checksum (test_attachments.AttachmentTest.test_png_and_jpeg_roundtrip_with_checksum) ... ok +test_too_large_is_413_and_nothing_is_stored (test_attachments.AttachmentTest.test_too_large_is_413_and_nothing_is_stored) ... ok + +---------------------------------------------------------------------- +Ran 11 tests in 5.449s + +OK +exit=0 + +$ python -m unittest discover -s student-tests -p test_topology.py -v +test_api_runs_as_non_root (test_topology.TopologyTest.test_api_runs_as_non_root) ... ok +test_at_least_two_api_replicas_without_published_ports (test_topology.TopologyTest.test_at_least_two_api_replicas_without_published_ports) ... ok +test_no_fixed_container_names (test_topology.TopologyTest.test_no_fixed_container_names) ... ok +test_only_lb_is_published_and_only_on_localhost (test_topology.TopologyTest.test_only_lb_is_published_and_only_on_localhost) ... ok +test_resource_limits_are_set (test_topology.TopologyTest.test_resource_limits_are_set) ... ok +test_stop_grace_period_covers_drain (test_topology.TopologyTest.test_stop_grace_period_covers_drain) ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 2.739s + +OK +exit=0 + +$ python -m unittest discover -s student-tests -p test_pipeline.py -v +test_alpha_is_composited_on_white (test_pipeline.PipelineTest.test_alpha_is_composited_on_white) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab6\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab6\\student-tests\\fixtures\\alpha.png'> + return open(os.path.join(FIX, name), "rb").read() +ResourceWarning: Enable tracemalloc to get the object allocation traceback +ok +test_content_is_409_while_not_ready_and_original_is_untouched (test_pipeline.PipelineTest.test_content_is_409_while_not_ready_and_original_is_untouched) ... ok +test_exif_orientation_is_applied (test_pipeline.PipelineTest.test_exif_orientation_is_applied) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab6\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab6\\student-tests\\fixtures\\exif_rot6.jpg'> + return open(os.path.join(FIX, name), "rb").read() +ResourceWarning: Enable tracemalloc to get the object allocation traceback +ok +test_message_can_reference_attachment_before_ready_and_failed_cannot (test_pipeline.PipelineTest.test_message_can_reference_attachment_before_ready_and_failed_cannot) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab6\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab6\\student-tests\\fixtures\\poison.png'> + return open(os.path.join(FIX, name), "rb").read() +ResourceWarning: Enable tracemalloc to get the object allocation traceback +ok +test_other_variants_visible_only_to_chat_members (test_pipeline.PipelineTest.test_other_variants_visible_only_to_chat_members) ... ok +test_ready_job_redelivery_does_not_duplicate_variants (test_pipeline.PipelineTest.test_ready_job_redelivery_does_not_duplicate_variants) ... ok +test_sample_png_produces_all_variants_per_contract (test_pipeline.PipelineTest.test_sample_png_produces_all_variants_per_contract) ... ok +test_small_image_is_not_upscaled_but_crop_is_128 (test_pipeline.PipelineTest.test_small_image_is_not_upscaled_but_crop_is_128) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab6\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab6\\student-tests\\fixtures\\small.png'> + return open(os.path.join(FIX, name), "rb").read() +ResourceWarning: Enable tracemalloc to get the object allocation traceback +ok +test_staging_is_cleaned_after_ready_and_kept_for_failed (test_pipeline.PipelineTest.test_staging_is_cleaned_after_ready_and_kept_for_failed) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 10.668s + +OK +exit=0 + +$ python -m unittest discover -s student-tests -p test_observability.py -v +test_logs_are_structured_and_linked_by_trace_id (test_observability.ObservabilityTest.test_logs_are_structured_and_linked_by_trace_id) ... ok +test_metrics_for_http_jobs_queue_pool_worker (test_observability.ObservabilityTest.test_metrics_for_http_jobs_queue_pool_worker) ... ok +test_no_high_cardinality_or_secret_labels (test_observability.ObservabilityTest.test_no_high_cardinality_or_secret_labels) ... ok +test_one_trace_spans_api_outbox_broker_and_worker (test_observability.ObservabilityTest.test_one_trace_spans_api_outbox_broker_and_worker) ... ok +test_secrets_and_message_bodies_are_not_exported (test_observability.ObservabilityTest.test_secrets_and_message_bodies_are_not_exported) ... ok + +---------------------------------------------------------------------- +Ran 5 tests in 9.932s + +OK +exit=0 + +$ python -m unittest discover -s student-tests -p test_observability_retry.py -v +test_all_attempts_in_one_trace_and_logs (test_observability_retry.RetryTraceTest.test_all_attempts_in_one_trace_and_logs) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 19.482s + +OK +exit=0 + +$ python -m unittest discover -s student-tests -p test_reconcile5.py -v +test_ready_variants_and_uncommitted_side_effect_are_preserved (test_reconcile5.ReconcilePipelineTest.test_ready_variants_and_uncommitted_side_effect_are_preserved) ... ok +test_staging_is_protected_and_unowned_orphan_obeys_grace (test_reconcile5.ReconcilePipelineTest.test_staging_is_protected_and_unowned_orphan_obeys_grace) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 34.423s + +OK +exit=0 + +$ python -m unittest discover -s student-tests -p test_resilience5.py -v +test_1_worker_down_upload_api_restart_worker_up (test_resilience5.ResilienceTest.test_1_worker_down_upload_api_restart_worker_up) ... ok +test_2_broker_restart_keeps_accepted_jobs (test_resilience5.ResilienceTest.test_2_broker_restart_keeps_accepted_jobs) ... ok +test_3_broker_down_during_acceptance_outbox_delivers_later (test_resilience5.ResilienceTest.test_3_broker_down_during_acceptance_outbox_delivers_later) ... ok +test_4_api_crash_between_commit_and_publish (test_resilience5.ResilienceTest.test_4_api_crash_between_commit_and_publish) ... ok +test_5_worker_crash_after_s3_before_commit_lease_recovery (test_resilience5.ResilienceTest.test_5_worker_crash_after_s3_before_commit_lease_recovery) ... ok +test_6_worker_crash_after_commit_before_ack_redelivery_is_noop (test_resilience5.ResilienceTest.test_6_worker_crash_after_commit_before_ack_redelivery_is_noop) ... ok +test_7_retry_exhaustion_registry_and_operator_replay (test_resilience5.ResilienceTest.test_7_retry_exhaustion_registry_and_operator_replay) ... ok + +---------------------------------------------------------------------- +Ran 7 tests in 146.730s + +OK +exit=0 + +$ python -m unittest discover -s student-tests -p test_expiry.py -v +test_expiry_is_not_extended_by_use (test_expiry.ExpiryTest.test_expiry_is_not_extended_by_use) ... ok +test_session_expires_after_ttl (test_expiry.ExpiryTest.test_session_expires_after_ttl) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 21.421s + +OK +exit=0 + +$ python -m unittest discover -s student-tests -p test_dependencies.py -v +test_postgres_down_gives_bounded_503_never_anonymous_access (test_dependencies.DependencyTest.test_postgres_down_gives_bounded_503_never_anonymous_access) ... ok +test_valkey_down_is_bypassed (test_dependencies.DependencyTest.test_valkey_down_is_bypassed) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 21.620s + +OK +exit=0 + +$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +FAIL: restart hook failed (inspect your script locally) +exit=1 diff --git a/evidence/grafana_provisioning.txt b/evidence/grafana_provisioning.txt new file mode 100644 index 0000000..1332e40 --- /dev/null +++ b/evidence/grafana_provisioning.txt @@ -0,0 +1,35 @@ +$ curl -u admin:*** http://127.0.0.1:3000/api/search (dashboard импортирован provisioning-ом, а не вручную) + dash-folder Messenger eg028qble26m8e + dash-db Messenger: API, очередь, worker messenger-overview +$ curl -u admin:*** .../api/datasources (datasources из infra/grafana/provisioning) + Loki loki loki provisioned= True + Prometheus prometheus prometheus provisioned= True + Tempo tempo tempo provisioned= True + health prometheus: OK + health loki: OK + health tempo: OK +$ curl -u admin:*** .../api/dashboards/uid/messenger-overview (панели) + provisioned: True | ïàíåëåé: 22 + - RPS по маршрутам + - Доля ошибок 5xx (SLI) + - Latency p50 / p95 / p99 + - RPS по статус-кодам + - Latency p95 по маршрутам + - DB pool: занято / простаивает / ожидают + - Задания по исходам + - Длительность обработки p50 / p95 (worker) + - Время до ready p50 / p95 + - Глубина очереди (queued + processing) + - Возраст старейшей задачи + - Worker: занято / ёмкость + - RSS по сервисам + - CPU по сервисам (ядра) + - Collector: очередь экспорта и отказы + - Структурные логи (API и worker), ошибки и предупреждения + - Алерты (firing / pending) + - Последние трассы API (Tempo) +$ анонимный доступ выключен: + GET /api/search áåç ïàðîëÿ -> HTTP 401 +$ порты наружу (docker compose ps): +grafana 127.0.0.1:3000->3000/tcp +lb 127.0.0.1:8080->8080/tcp diff --git a/evidence/incidents.json b/evidence/incidents.json new file mode 100644 index 0000000..8e9bcd7 --- /dev/null +++ b/evidence/incidents.json @@ -0,0 +1,1727 @@ +{ + "date": "2026-10-03 04:15:12", + "traffic_counters": { + "msg_2": 17863, + "up_2": 12054, + "up_5": 2236, + "up_err": 48, + "up_4": 37 + }, + "incidents": [ + { + "incident": "worker_down", + "alert": "QueueLagGrowing", + "injected_at_s": 25.0, + "firing_at_s": 88.8, + "detection_seconds": 63.8, + "recovered_cmd_at_s": 91.6, + "resolved_at_s": 115.5, + "fired": true, + "resolved": true, + "evidence": { + "queue_oldest_age_s": 56.708083, + "worker_capacity_series_present": false, + "jobs_ready_rate_1m": 0.0, + "diagnosis": "возраст старейшей задачи растёт при нулевой скорости обработки jobs (worker не отдаёт метрики) -> остановлен worker, а не медленная БД", + "alerts_in_state_firing": [ + "HighHttpErrorRate", + "HighUploadErrorRate", + "QueueLagGrowing", + "WorkerSilent" + ] + }, + "time_to_resolve_after_recovery_s": 24.0 + }, + { + "incident": "slow_db", + "alert": "HighLatencyP95", + "injected_at_s": 121.5, + "firing_at_s": 193.4, + "detection_seconds": 71.9, + "recovered_cmd_at_s": 194.7, + "resolved_at_s": 240.3, + "fired": true, + "resolved": true, + "evidence": { + "p95_s": 0.8010330578512395, + "db_pool_waiting": 0.0, + "slow_traces_with_long_db_span": 20, + "slow_http_post_messages_traces": 20, + "diagnosis": "p95 высокий, а время в спанах db.connection сопоставимо с длительностью запроса и растёт ожидание пула -> медленная БД, а не обработчик", + "alerts_in_state_firing": [ + "HighLatencyP95" + ] + }, + "time_to_resolve_after_recovery_s": 45.7 + }, + { + "incident": "s3_down", + "alert": "HighUploadErrorRate", + "injected_at_s": 246.3, + "firing_at_s": 334.5, + "detection_seconds": 88.2, + "recovered_cmd_at_s": 335.9, + "resolved_at_s": 414.1, + "fired": true, + "resolved": true, + "evidence": { + "upload_error_ratio_1m": 0.32, + "global_error_ratio_1m": 0.008221993833504625, + "error_traces": 20, + "s3_put_failed_log_lines": 139, + "diagnosis": "рост доли 5xx на маршруте загрузки при нормальных остальных; спаны s3.put_object с ошибкой и логи s3_put_failed -> недоступен S3", + "alerts_in_state_firing": [ + "HighUploadErrorRate", + "QueueLagGrowing" + ] + }, + "time_to_resolve_after_recovery_s": 78.2 + }, + { + "incident": "telemetry_backend", + "alert": "TelemetryExportFailing", + "injected_at_s": 420.1, + "firing_at_s": 469.1, + "detection_seconds": 48.9, + "recovered_cmd_at_s": 470.6, + "resolved_at_s": 555.5, + "fired": true, + "resolved": true, + "evidence": { + "api_error_ratio_1m": 0.0, + "api_rps": 67.0194003527337, + "collector_queue_size": 23.0, + "collector_send_failed_spans_rate": 65.749040856774, + "collector_rss_mib": 247.1, + "diagnosis": "ошибки экспорта растут, очередь ограничена; бизнес-метрики API без роста ошибок -> сбой хранилища телеметрии, не приложения", + "alerts_in_state_firing": [ + "TelemetryExportFailing" + ] + }, + "time_to_resolve_after_recovery_s": 84.9 + } + ], + "timeline": [ + { + "t": 0.0, + "alerts": {}, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.0, + "p95_s": 0.021837988516905667, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 0.6909342157896651 + }, + { + "t": 4.1, + "alerts": {}, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.0, + "p95_s": 0.02183799436386364, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 0.6909467789152136 + }, + { + "t": 8.3, + "alerts": {}, + "queue_oldest_age_s": 0.202415, + "error_ratio_1m": 0.0, + "p95_s": 0.02183799436386364, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 0.6909467789152136 + }, + { + "t": 12.5, + "alerts": {}, + "queue_oldest_age_s": 0.032457, + "error_ratio_1m": 0.0, + "p95_s": 0.02183799436386364, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 0.6909342157896651 + }, + { + "t": 16.6, + "alerts": {}, + "queue_oldest_age_s": 0.202396, + "error_ratio_1m": 0.0, + "p95_s": 0.08750000000000001, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 3.454733894576068 + }, + { + "t": 20.8, + "alerts": {}, + "queue_oldest_age_s": 0.005507, + "error_ratio_1m": 0.0, + "p95_s": 0.031249999999999972, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 9.455061185155554 + }, + { + "t": 24.9, + "alerts": {}, + "queue_oldest_age_s": 0.005507, + "error_ratio_1m": 0.0, + "p95_s": 0.031366279069767425, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 15.655114731444781 + }, + { + "t": 29.0, + "alerts": {}, + "queue_oldest_age_s": 0.172278, + "error_ratio_1m": 0.0, + "p95_s": 0.031307692307692286, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 21.674303585715325 + }, + { + "t": 33.1, + "alerts": {}, + "queue_oldest_age_s": 0.669169, + "error_ratio_1m": 0.0, + "p95_s": 0.031307692307692286, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 21.674303585715325 + }, + { + "t": 37.3, + "alerts": {}, + "queue_oldest_age_s": 6.67272, + "error_ratio_1m": 0.0, + "p95_s": 0.028216666666666647, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 27.7646459871268 + }, + { + "t": 41.4, + "alerts": {}, + "queue_oldest_age_s": 10.674385, + "error_ratio_1m": 0.0, + "p95_s": 0.025135542168674578, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 34.02003745658854 + }, + { + "t": 45.5, + "alerts": {}, + "queue_oldest_age_s": 16.677877, + "error_ratio_1m": 0.00908265213442325, + "p95_s": 0.024859863730645866, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 40.0378195570748 + }, + { + "t": 49.6, + "alerts": {}, + "queue_oldest_age_s": 16.677877, + "error_ratio_1m": 0.07276341948310139, + "p95_s": 0.02458815658966605, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 45.728935597658094 + }, + { + "t": 53.8, + "alerts": { + "HighHttpErrorRate": "pending" + }, + "queue_oldest_age_s": 20.679374, + "error_ratio_1m": 0.07276341948310139, + "p95_s": 0.02458815658966605, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 51.855488281605126 + }, + { + "t": 57.9, + "alerts": { + "HighHttpErrorRate": "pending" + }, + "queue_oldest_age_s": 26.681594, + "error_ratio_1m": 0.12096774193548386, + "p95_s": 0.02433063583815029, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 51.855488281605126 + }, + { + "t": 62.0, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "pending", + "WorkerSilent": "pending" + }, + "queue_oldest_age_s": 30.682897, + "error_ratio_1m": 0.15879962488277585, + "p95_s": 0.024032006920415225, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 58.166809098678115 + }, + { + "t": 66.2, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "pending", + "WorkerSilent": "pending" + }, + "queue_oldest_age_s": 36.686688, + "error_ratio_1m": 0.18910527801298335, + "p95_s": 0.02373442906574394, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 64.42286711760855 + }, + { + "t": 70.3, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "pending", + "QueueLagGrowing": "pending", + "WorkerSilent": "pending" + }, + "queue_oldest_age_s": 40.688213, + "error_ratio_1m": 0.22231139646869982, + "p95_s": 0.023133872976338746, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.96734367329128 + }, + { + "t": 74.5, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "pending", + "WorkerSilent": "pending" + }, + "queue_oldest_age_s": 40.688213, + "error_ratio_1m": 0.2655457699492928, + "p95_s": 0.022416792168674704, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.12975017273357 + }, + { + "t": 78.6, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "pending", + "WorkerSilent": "pending" + }, + "queue_oldest_age_s": 46.691598, + "error_ratio_1m": 0.2655457699492928, + "p95_s": 0.022416792168674704, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.3297574457253 + }, + { + "t": 82.7, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "pending", + "WorkerSilent": "pending" + }, + "queue_oldest_age_s": 50.693747, + "error_ratio_1m": 0.3084087280468334, + "p95_s": 0.021089792060491497, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.3297574457253 + }, + { + "t": 86.8, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "pending", + "WorkerSilent": "pending" + }, + "queue_oldest_age_s": 56.708083, + "error_ratio_1m": 0.35022552401167417, + "p95_s": 0.019042698019801962, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.52851870034002 + }, + { + "t": 90.9, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing", + "WorkerSilent": "firing" + }, + "queue_oldest_age_s": 60.708464, + "error_ratio_1m": 0.39262990455991514, + "p95_s": 0.015761732851985573, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.58555921232067 + }, + { + "t": 95.0, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing", + "WorkerSilent": "firing" + }, + "queue_oldest_age_s": 60.708464, + "error_ratio_1m": 0.43625761993108936, + "p95_s": 0.009811190476190473, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.60249463616857 + }, + { + "t": 99.2, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing", + "WorkerSilent": "firing" + }, + "queue_oldest_age_s": 66.710115, + "error_ratio_1m": 0.4717379820390913, + "p95_s": 0.00916851106639839, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.84011855192102 + }, + { + "t": 103.4, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing", + "WorkerSilent": "firing" + }, + "queue_oldest_age_s": 66.176169, + "error_ratio_1m": 0.4717379820390913, + "p95_s": 0.00916851106639839, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 68.84011855192102 + }, + { + "t": 107.5, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing", + "WorkerSilent": "firing" + }, + "queue_oldest_age_s": 2.63989, + "error_ratio_1m": 0.4717529039070751, + "p95_s": 0.00910020876826722, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 68.8777365626591 + }, + { + "t": 111.7, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 0.881435, + "error_ratio_1m": 0.43095679830373707, + "p95_s": 0.009858588351431389, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.60374202229212 + }, + { + "t": 115.8, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.11786, + "error_ratio_1m": 0.388682252922423, + "p95_s": 0.015365187713310564, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.4376079565083 + }, + { + "t": 120.0, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.11786, + "error_ratio_1m": 0.346194784459819, + "p95_s": 0.01859507042253521, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 68.33099987272034 + }, + { + "t": 124.1, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.229302, + "error_ratio_1m": 0.30473135525260625, + "p95_s": 0.020374999999999983, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 68.02065529655624 + }, + { + "t": 128.3, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.149023, + "error_ratio_1m": 0.30473135525260625, + "p95_s": 0.020374999999999983, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 68.02065529655624 + }, + { + "t": 132.4, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.2611021936864634, + "p95_s": 0.021397841726618707, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 67.96734367329127 + }, + { + "t": 136.7, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.06846, + "error_ratio_1m": 0.21922246220302374, + "p95_s": 0.02243272727272727, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 67.34912813426187 + }, + { + "t": 140.9, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.15416, + "error_ratio_1m": 0.18384637108161536, + "p95_s": 0.02345556133056133, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 64.38415942397906 + }, + { + "t": 145.2, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.206758, + "error_ratio_1m": 0.14522821576763487, + "p95_s": 0.024283271719038817, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 61.34768537037711 + }, + { + "t": 149.6, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.206758, + "error_ratio_1m": 0.10130922693266832, + "p95_s": 0.024884331419196062, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 58.32939379613804 + }, + { + "t": 153.8, + "alerts": { + "HighHttpErrorRate": "firing" + }, + "queue_oldest_age_s": 0.245449, + "error_ratio_1m": 0.10130922693266832, + "p95_s": 0.024884331419196062, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 55.23937669327418 + }, + { + "t": 158.0, + "alerts": { + "HighHttpErrorRate": "firing" + }, + "queue_oldest_age_s": 0.055453, + "error_ratio_1m": 0.054312047399605004, + "p95_s": 0.035510752688172026, + "db_waiting": 0.0, + "db_used": 6.0, + "rps": 55.23937669327418 + }, + { + "t": 162.2, + "alerts": { + "HighHttpErrorRate": "firing" + }, + "queue_oldest_age_s": 0.114868, + "error_ratio_1m": 0.00034867503486750353, + "p95_s": 0.04513636363636352, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 52.14640266186658 + }, + { + "t": 166.3, + "alerts": {}, + "queue_oldest_age_s": 0.203497, + "error_ratio_1m": 0.0, + "p95_s": 0.08299999999999742, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 49.23815411469508 + }, + { + "t": 170.4, + "alerts": {}, + "queue_oldest_age_s": 0.006329, + "error_ratio_1m": 0.0, + "p95_s": 0.5865259740259738, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 46.31161699729076 + }, + { + "t": 174.6, + "alerts": { + "HighLatencyP95": "pending" + }, + "queue_oldest_age_s": 0.006329, + "error_ratio_1m": 0.0, + "p95_s": 0.6610795454545451, + "db_waiting": 0.0, + "db_used": 6.0, + "rps": 43.38260695649012 + }, + { + "t": 178.7, + "alerts": { + "HighLatencyP95": "pending" + }, + "queue_oldest_age_s": 0.048792, + "error_ratio_1m": 0.0, + "p95_s": 0.6610795454545451, + "db_waiting": 0.0, + "db_used": 6.0, + "rps": 40.51130061639726 + }, + { + "t": 182.8, + "alerts": { + "HighLatencyP95": "pending" + }, + "queue_oldest_age_s": 0.056091, + "error_ratio_1m": 0.0, + "p95_s": 0.7186868686868686, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 40.51130061639726 + }, + { + "t": 187.0, + "alerts": { + "HighLatencyP95": "pending" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.0, + "p95_s": 0.7653409090909089, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 37.54681988435943 + }, + { + "t": 191.1, + "alerts": { + "HighLatencyP95": "pending" + }, + "queue_oldest_age_s": 0.105751, + "error_ratio_1m": 0.0, + "p95_s": 0.8010330578512395, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 35.019455252918284 + }, + { + "t": 195.2, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.200896, + "error_ratio_1m": 0.0, + "p95_s": 0.7996875000000001, + "db_waiting": 0.0, + "db_used": 7.0, + "rps": 34.96490781482963 + }, + { + "t": 199.3, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.200896, + "error_ratio_1m": 0.0, + "p95_s": 0.7991666666666664, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 35.05518282150585 + }, + { + "t": 203.6, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.005269, + "error_ratio_1m": 0.0, + "p95_s": 0.7991666666666664, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 35.05582021164406 + }, + { + "t": 207.8, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.127232, + "error_ratio_1m": 0.0, + "p95_s": 0.7991666666666662, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 35.05582021164406 + }, + { + "t": 211.9, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.238371, + "error_ratio_1m": 0.0, + "p95_s": 0.7834051724137925, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 36.546783519400705 + }, + { + "t": 216.0, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.128177, + "error_ratio_1m": 0.0, + "p95_s": 0.7396634615384619, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 39.383966398167175 + }, + { + "t": 220.1, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.230894, + "error_ratio_1m": 0.0, + "p95_s": 0.690558510638298, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 42.31062947743554 + }, + { + "t": 224.2, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.230894, + "error_ratio_1m": 0.0, + "p95_s": 0.6250000000000003, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 45.27437361358595 + }, + { + "t": 228.4, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.157193, + "error_ratio_1m": 0.0, + "p95_s": 0.6250000000000003, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 48.255422825869566 + }, + { + "t": 232.6, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.212977, + "error_ratio_1m": 0.0, + "p95_s": 0.5392361111111098, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 48.255422825869566 + }, + { + "t": 236.8, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.067636, + "error_ratio_1m": 0.0, + "p95_s": 0.048266129032258034, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 51.23729522354952 + }, + { + "t": 240.9, + "alerts": {}, + "queue_oldest_age_s": 0.188142, + "error_ratio_1m": 0.0, + "p95_s": 0.04448717948717947, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 54.184773714929904 + }, + { + "t": 245.1, + "alerts": {}, + "queue_oldest_age_s": 0.006211, + "error_ratio_1m": 0.0, + "p95_s": 0.040921717171717165, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 57.057657690419475 + }, + { + "t": 249.3, + "alerts": {}, + "queue_oldest_age_s": 0.006211, + "error_ratio_1m": 0.0, + "p95_s": 0.03712752525252522, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 59.98509009582341 + }, + { + "t": 253.4, + "alerts": {}, + "queue_oldest_age_s": 0.112589, + "error_ratio_1m": 0.0, + "p95_s": 0.03712752525252522, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 59.98509009582341 + }, + { + "t": 257.5, + "alerts": {}, + "queue_oldest_age_s": 5.809907, + "error_ratio_1m": 0.0, + "p95_s": 0.03351522842639587, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 62.87615688128443 + }, + { + "t": 261.6, + "alerts": {}, + "queue_oldest_age_s": 9.815523, + "error_ratio_1m": 0.0002787068004459309, + "p95_s": 0.02810439560439558, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 65.23873595403468 + }, + { + "t": 265.7, + "alerts": {}, + "queue_oldest_age_s": 15.820854, + "error_ratio_1m": 0.0022785531187695814, + "p95_s": 0.025375766871165672, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 63.837524318623984 + }, + { + "t": 269.8, + "alerts": {}, + "queue_oldest_age_s": 15.820854, + "error_ratio_1m": 0.0029726516052318666, + "p95_s": 0.024835956917978463, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 61.16586057674825 + }, + { + "t": 273.9, + "alerts": {}, + "queue_oldest_age_s": 19.823123, + "error_ratio_1m": 0.0029726516052318666, + "p95_s": 0.024835956917978463, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 58.422430722234346 + }, + { + "t": 278.0, + "alerts": {}, + "queue_oldest_age_s": 25.825538, + "error_ratio_1m": 0.004668534080298786, + "p95_s": 0.02458166515013649, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 58.422430722234346 + }, + { + "t": 282.1, + "alerts": {}, + "queue_oldest_age_s": 29.827643, + "error_ratio_1m": 0.005584756898817346, + "p95_s": 0.02436680327868853, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 55.3484735531029 + }, + { + "t": 286.2, + "alerts": {}, + "queue_oldest_age_s": 35.830777, + "error_ratio_1m": 0.005923344947735192, + "p95_s": 0.024117122473246128, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 52.18466461807007 + }, + { + "t": 290.3, + "alerts": {}, + "queue_oldest_age_s": 39.831666, + "error_ratio_1m": 0.006649427410417436, + "p95_s": 0.023801063829787236, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 49.2199716353322 + }, + { + "t": 294.4, + "alerts": { + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 39.831666, + "error_ratio_1m": 0.00706713780918728, + "p95_s": 0.023138695652173913, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 46.30993290787106 + }, + { + "t": 298.5, + "alerts": { + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 45.833089, + "error_ratio_1m": 0.00706713780918728, + "p95_s": 0.023138695652173913, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 43.34703080112004 + }, + { + "t": 302.6, + "alerts": { + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 49.835019, + "error_ratio_1m": 0.00964765100671141, + "p95_s": 0.02236995515695067, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 43.34703080112004 + }, + { + "t": 306.7, + "alerts": { + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 55.838342, + "error_ratio_1m": 0.011230907457322551, + "p95_s": 0.0215762987012987, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 40.47419906178407 + }, + { + "t": 310.8, + "alerts": { + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 59.840128, + "error_ratio_1m": 0.012100677637947725, + "p95_s": 0.019078034682080913, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 37.565685401021874 + }, + { + "t": 314.9, + "alerts": { + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 59.840128, + "error_ratio_1m": 0.01295336787564767, + "p95_s": 0.009926075268817204, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 35.09282324490427 + }, + { + "t": 319.0, + "alerts": { + "HighUploadErrorRate": "pending", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 65.842317, + "error_ratio_1m": 0.01295336787564767, + "p95_s": 0.009962319494584837, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 35.037637732281176 + }, + { + "t": 323.1, + "alerts": { + "HighUploadErrorRate": "pending", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 69.842659, + "error_ratio_1m": 0.011935651271406332, + "p95_s": 0.009962319494584837, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 35.037637732281176 + }, + { + "t": 327.3, + "alerts": { + "HighUploadErrorRate": "pending", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 75.844071, + "error_ratio_1m": 0.0109204368174727, + "p95_s": 0.00995516784452297, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 34.964272077674146 + }, + { + "t": 331.4, + "alerts": { + "HighUploadErrorRate": "pending", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 79.846203, + "error_ratio_1m": 0.009375, + "p95_s": 0.009844827586206896, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 34.910360376740975 + }, + { + "t": 335.6, + "alerts": { + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 85.848987, + "error_ratio_1m": 0.008221993833504625, + "p95_s": 0.00977725724020443, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 35.38310484017601 + }, + { + "t": 339.7, + "alerts": { + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 85.848987, + "error_ratio_1m": 0.008646998982706001, + "p95_s": 0.00984141156462585, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 35.74675442743373 + }, + { + "t": 343.9, + "alerts": { + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 89.850732, + "error_ratio_1m": 0.008646998982706001, + "p95_s": 0.00984141156462585, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 36.219498890868756 + }, + { + "t": 348.0, + "alerts": { + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.010542168674698798, + "p95_s": 0.009932758620689654, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 36.219498890868756 + }, + { + "t": 352.2, + "alerts": { + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.010558069381598792, + "p95_s": 0.009950518582541054, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 36.165609033219994 + }, + { + "t": 356.3, + "alerts": { + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.015888778550148957, + "p95_s": 0.013761538461538462, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 36.61951343685225 + }, + { + "t": 360.5, + "alerts": { + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.02747791952894995, + "p95_s": 0.018929687500000014, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 37.05589294156152 + }, + { + "t": 364.7, + "alerts": { + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.08081746400371574, + "p95_s": 0.03874999999999977, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 39.1468780682934 + }, + { + "t": 368.8, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.153313, + "error_ratio_1m": 0.08081746400371574, + "p95_s": 0.03874999999999977, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 41.34695807120259 + }, + { + "t": 373.0, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.12884784520668424, + "p95_s": 0.032499999999999814, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 41.34695807120259 + }, + { + "t": 377.2, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.121416, + "error_ratio_1m": 0.12050209205020922, + "p95_s": 0.032065217391304184, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 43.45691583177264 + }, + { + "t": 381.5, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.184617, + "error_ratio_1m": 0.1131630648330059, + "p95_s": 0.029094827586206743, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 46.27525137734786 + }, + { + "t": 385.8, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.009728, + "error_ratio_1m": 0.10565201329885482, + "p95_s": 0.02490677179962894, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 49.21818181818181 + }, + { + "t": 390.0, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.009728, + "error_ratio_1m": 0.10056258790436005, + "p95_s": 0.024884101040118872, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 51.71097130804757 + }, + { + "t": 394.1, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.121893, + "error_ratio_1m": 0.09541345831938401, + "p95_s": 0.024956211180124213, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 54.31304094843261 + }, + { + "t": 398.3, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.09541345831938401, + "p95_s": 0.024956211180124213, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 54.31304094843261 + }, + { + "t": 402.5, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.118105, + "error_ratio_1m": 0.0896, + "p95_s": 0.0249156852248394, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 56.822314350134555 + }, + { + "t": 406.6, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.181903, + "error_ratio_1m": 0.08515815085158152, + "p95_s": 0.026874999999999916, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 59.78507918613742 + }, + { + "t": 410.7, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.026519, + "error_ratio_1m": 0.07708029197080292, + "p95_s": 0.025170454545454603, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 62.27612415222649 + }, + { + "t": 414.9, + "alerts": { + "HighHttpErrorRate": "firing" + }, + "queue_oldest_age_s": 0.026519, + "error_ratio_1m": 0.06691031768344109, + "p95_s": 0.024704908675799084, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 64.67507909378523 + }, + { + "t": 419.0, + "alerts": { + "HighHttpErrorRate": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.06691031768344109, + "p95_s": 0.024266327247191008, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 65.69329793810684 + }, + { + "t": 423.2, + "alerts": { + "HighHttpErrorRate": "firing" + }, + "queue_oldest_age_s": 0.11007, + "error_ratio_1m": 0.03321339606974814, + "p95_s": 0.024266327247191008, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 65.69329793810684 + }, + { + "t": 427.3, + "alerts": {}, + "queue_oldest_age_s": 0.025811, + "error_ratio_1m": 0.0, + "p95_s": 0.024248980978260865, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 66.43877959198517 + }, + { + "t": 431.6, + "alerts": {}, + "queue_oldest_age_s": 0.117773, + "error_ratio_1m": 0.0, + "p95_s": 0.024118148395721926, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 67.25821408440461 + }, + { + "t": 435.8, + "alerts": {}, + "queue_oldest_age_s": 0.058093, + "error_ratio_1m": 0.0, + "p95_s": 0.02405114401076716, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.27639689437606 + }, + { + "t": 439.9, + "alerts": {}, + "queue_oldest_age_s": 0.058093, + "error_ratio_1m": 0.0, + "p95_s": 0.02399829816201498, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.31398647174342 + }, + { + "t": 444.1, + "alerts": {}, + "queue_oldest_age_s": 0.171888, + "error_ratio_1m": 0.0, + "p95_s": 0.023994866529774124, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.23758613792978 + }, + { + "t": 448.3, + "alerts": {}, + "queue_oldest_age_s": 0.117493, + "error_ratio_1m": 0.0, + "p95_s": 0.023994866529774124, + "db_waiting": 0.0, + "db_used": 6.0, + "rps": 67.23758613792978 + }, + { + "t": 452.4, + "alerts": {}, + "queue_oldest_age_s": 0.165172, + "error_ratio_1m": 0.0, + "p95_s": 0.02398979591836735, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.27395043546247 + }, + { + "t": 456.5, + "alerts": { + "TelemetryExportFailing": "pending" + }, + "queue_oldest_age_s": 0.091433, + "error_ratio_1m": 0.0, + "p95_s": 0.024048507462686566, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.09212894779905 + }, + { + "t": 460.7, + "alerts": { + "TelemetryExportFailing": "pending" + }, + "queue_oldest_age_s": 0.16713, + "error_ratio_1m": 0.0, + "p95_s": 0.024109723160027005, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.23758613792978 + }, + { + "t": 464.8, + "alerts": { + "TelemetryExportFailing": "pending" + }, + "queue_oldest_age_s": 0.16713, + "error_ratio_1m": 0.0, + "p95_s": 0.024107479784366576, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 67.0194003527337 + }, + { + "t": 469.1, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.060032, + "error_ratio_1m": 0.0, + "p95_s": 0.02404714765100671, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 67.14911722457589 + }, + { + "t": 473.2, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.0434, + "error_ratio_1m": 0.0, + "p95_s": 0.02404714765100671, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 67.14911722457589 + }, + { + "t": 477.5, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.027894, + "error_ratio_1m": 0.0, + "p95_s": 0.024051839464882943, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.09334884904906 + }, + { + "t": 481.6, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.113299, + "error_ratio_1m": 0.0, + "p95_s": 0.024043224299065417, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.11275160463298 + }, + { + "t": 485.8, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.039427, + "error_ratio_1m": 0.0, + "p95_s": 0.024047981469225675, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.23880868395214 + }, + { + "t": 490.0, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.039427, + "error_ratio_1m": 0.0, + "p95_s": 0.024082046332046324, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 67.31153860140368 + }, + { + "t": 494.2, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.166758, + "error_ratio_1m": 0.0, + "p95_s": 0.024164996869129617, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 67.23880868395214 + }, + { + "t": 498.4, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.036081, + "error_ratio_1m": 0.0, + "p95_s": 0.024164996869129617, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.42185937414769 + }, + { + "t": 502.6, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.136676, + "error_ratio_1m": 0.0, + "p95_s": 0.02413483146067416, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.42185937414769 + }, + { + "t": 506.8, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.113007, + "error_ratio_1m": 0.0, + "p95_s": 0.024090405319822678, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.54791083312121 + }, + { + "t": 511.0, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.0, + "p95_s": 0.02406825447570332, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.69460152371947 + }, + { + "t": 515.2, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.159875, + "error_ratio_1m": 0.0, + "p95_s": 0.02383058898036732, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.67641871374802 + }, + { + "t": 519.5, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.159875, + "error_ratio_1m": 0.0, + "p95_s": 0.023655025284450064, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.94916086331982 + }, + { + "t": 523.6, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.0, + "p95_s": 0.023655025284450064, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.92974289974181 + }, + { + "t": 527.8, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.133987, + "error_ratio_1m": 0.0, + "p95_s": 0.023759013282732448, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 67.92974289974181 + }, + { + "t": 532.1, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.0, + "p95_s": 0.023960063897763582, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.8206480235645 + }, + { + "t": 536.3, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.134529, + "error_ratio_1m": 0.0, + "p95_s": 0.023997766432673893, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.89214349351806 + }, + { + "t": 540.6, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.205896, + "error_ratio_1m": 0.0, + "p95_s": 0.024080629770992363, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.76486845215368 + }, + { + "t": 544.9, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.205896, + "error_ratio_1m": 0.0, + "p95_s": 0.024150970873786412, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.80246554420161 + }, + { + "t": 549.1, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.123157, + "error_ratio_1m": 0.0, + "p95_s": 0.02423262739907346, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.74791810611295 + }, + { + "t": 553.3, + "alerts": {}, + "queue_oldest_age_s": 0.17385, + "error_ratio_1m": 0.0, + "p95_s": 0.02423262739907346, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.74791810611295 + }, + { + "t": 557.4, + "alerts": {}, + "queue_oldest_age_s": 0.076714, + "error_ratio_1m": 0.0, + "p95_s": 0.024358974358974363, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 67.63882322993564 + }, + { + "t": 561.6, + "alerts": {}, + "queue_oldest_age_s": 0.081785, + "error_ratio_1m": 0.0, + "p95_s": 0.024423342087984242, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 67.47518091566967 + } + ] +} \ No newline at end of file diff --git a/evidence/incidents.png b/evidence/incidents.png new file mode 100644 index 0000000..7b548d2 Binary files /dev/null and b/evidence/incidents.png differ diff --git a/evidence/obs_cost.json b/evidence/obs_cost.json new file mode 100644 index 0000000..de9a553 --- /dev/null +++ b/evidence/obs_cost.json @@ -0,0 +1,1377 @@ +{ + "date": "2026-10-03 04:53:16", + "args": { + "out": "evidence", + "profiles": [ + "off", + "on100", + "head10", + "tail", + "on100_bigq" + ], + "merge": false, + "seconds": 30, + "concurrency": 64, + "procs": 8, + "flush": 25, + "repeat": 2 + }, + "faults": "http500:0.02,slow:0.02:0.6", + "results": [ + { + "profile": "off", + "post": { + "requests": 40957, + "ok": 40097, + "errors": { + "500": 860 + }, + "seconds": 30, + "ok_rps": 1336.6, + "error_ratio": 0.021, + "p50_ms": 15.47, + "p95_ms": 74.45, + "p99_ms": 615.58 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.8, + "peak_mem_mib": 54.5 + }, + "api": { + "avg_cpu_percent": 53.3, + "peak_mem_mib": 76.3 + }, + "otel-collector": { + "avg_cpu_percent": 0.1, + "peak_mem_mib": 39.1 + }, + "tempo": { + "avg_cpu_percent": 0.3, + "peak_mem_mib": 506.8 + }, + "lb": { + "avg_cpu_percent": 23.1, + "peak_mem_mib": 25.2 + }, + "grafana": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 404.4 + }, + "rabbitmq": { + "avg_cpu_percent": 13.1, + "peak_mem_mib": 200.9 + }, + "postgres": { + "avg_cpu_percent": 99.8, + "peak_mem_mib": 368.7 + }, + "loki": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 474.8 + }, + "seaweedfs": { + "avg_cpu_percent": 1.5, + "peak_mem_mib": 546.5 + }, + "prometheus": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 74.1 + }, + "valkey": { + "avg_cpu_percent": 5.2, + "peak_mem_mib": 11.0 + } + }, + "collector_delta": { + "receiver_accepted_spans": 0, + "receiver_accepted_log_records": 0, + "receiver_accepted_metric_points": 0, + "receiver_refused_spans": 0, + "exporter_sent_spans": 0, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": -544768, + "loki_bytes_delta": 4280320, + "errors_injected_500": 875, + "slow_requests_gt500ms": 821, + "traces_in_tempo": { + "all_http": 0, + "error": 0, + "slow_gt500ms": 0 + }, + "kept_ratio": { + "errors": 0.0, + "slow": 0.0 + }, + "rps_runs": [ + 1344.6, + 1336.6 + ], + "p95_runs": [ + 73.8, + 74.45 + ], + "runs": [ + { + "profile": "off", + "post": { + "requests": 41134, + "ok": 40337, + "errors": { + "500": 797 + }, + "seconds": 30, + "ok_rps": 1344.6, + "error_ratio": 0.0194, + "p50_ms": 15.04, + "p95_ms": 73.8, + "p99_ms": 615.65 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.7, + "peak_mem_mib": 54.0 + }, + "api": { + "avg_cpu_percent": 52.4, + "peak_mem_mib": 76.3 + }, + "otel-collector": { + "avg_cpu_percent": 0.1, + "peak_mem_mib": 44.6 + }, + "tempo": { + "avg_cpu_percent": 0.4, + "peak_mem_mib": 507.1 + }, + "lb": { + "avg_cpu_percent": 23.5, + "peak_mem_mib": 24.3 + }, + "grafana": { + "avg_cpu_percent": 1.0, + "peak_mem_mib": 404.1 + }, + "rabbitmq": { + "avg_cpu_percent": 8.2, + "peak_mem_mib": 201.5 + }, + "postgres": { + "avg_cpu_percent": 100.2, + "peak_mem_mib": 370.8 + }, + "loki": { + "avg_cpu_percent": 1.1, + "peak_mem_mib": 484.6 + }, + "seaweedfs": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 546.2 + }, + "prometheus": { + "avg_cpu_percent": 0.5, + "peak_mem_mib": 73.7 + }, + "valkey": { + "avg_cpu_percent": 5.1, + "peak_mem_mib": 11.0 + } + }, + "collector_delta": { + "receiver_accepted_spans": 0, + "receiver_accepted_log_records": 0, + "receiver_accepted_metric_points": 0, + "receiver_refused_spans": 0, + "exporter_sent_spans": 0, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 249856, + "loki_bytes_delta": 3112960, + "errors_injected_500": 813, + "slow_requests_gt500ms": 863, + "traces_in_tempo": { + "all_http": 0, + "error": 0, + "slow_gt500ms": 0 + }, + "kept_ratio": { + "errors": 0.0, + "slow": 0.0 + } + }, + { + "profile": "off", + "post": { + "requests": 40957, + "ok": 40097, + "errors": { + "500": 860 + }, + "seconds": 30, + "ok_rps": 1336.6, + "error_ratio": 0.021, + "p50_ms": 15.47, + "p95_ms": 74.45, + "p99_ms": 615.58 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.8, + "peak_mem_mib": 54.5 + }, + "api": { + "avg_cpu_percent": 53.3, + "peak_mem_mib": 76.3 + }, + "otel-collector": { + "avg_cpu_percent": 0.1, + "peak_mem_mib": 39.1 + }, + "tempo": { + "avg_cpu_percent": 0.3, + "peak_mem_mib": 506.8 + }, + "lb": { + "avg_cpu_percent": 23.1, + "peak_mem_mib": 25.2 + }, + "grafana": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 404.4 + }, + "rabbitmq": { + "avg_cpu_percent": 13.1, + "peak_mem_mib": 200.9 + }, + "postgres": { + "avg_cpu_percent": 99.8, + "peak_mem_mib": 368.7 + }, + "loki": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 474.8 + }, + "seaweedfs": { + "avg_cpu_percent": 1.5, + "peak_mem_mib": 546.5 + }, + "prometheus": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 74.1 + }, + "valkey": { + "avg_cpu_percent": 5.2, + "peak_mem_mib": 11.0 + } + }, + "collector_delta": { + "receiver_accepted_spans": 0, + "receiver_accepted_log_records": 0, + "receiver_accepted_metric_points": 0, + "receiver_refused_spans": 0, + "exporter_sent_spans": 0, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": -544768, + "loki_bytes_delta": 4280320, + "errors_injected_500": 875, + "slow_requests_gt500ms": 821, + "traces_in_tempo": { + "all_http": 0, + "error": 0, + "slow_gt500ms": 0 + }, + "kept_ratio": { + "errors": 0.0, + "slow": 0.0 + } + } + ] + }, + { + "profile": "on100", + "post": { + "requests": 40983, + "ok": 40144, + "errors": { + "500": 839 + }, + "seconds": 30, + "ok_rps": 1338.1, + "error_ratio": 0.0205, + "p50_ms": 22.75, + "p95_ms": 70.25, + "p99_ms": 628.03 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.8, + "peak_mem_mib": 59.7 + }, + "api": { + "avg_cpu_percent": 86.9, + "peak_mem_mib": 87.9 + }, + "otel-collector": { + "avg_cpu_percent": 3.1, + "peak_mem_mib": 70.8 + }, + "tempo": { + "avg_cpu_percent": 6.3, + "peak_mem_mib": 389.0 + }, + "lb": { + "avg_cpu_percent": 22.3, + "peak_mem_mib": 24.5 + }, + "grafana": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 410.1 + }, + "rabbitmq": { + "avg_cpu_percent": 8.5, + "peak_mem_mib": 205.7 + }, + "postgres": { + "avg_cpu_percent": 96.5, + "peak_mem_mib": 373.0 + }, + "loki": { + "avg_cpu_percent": 4.9, + "peak_mem_mib": 497.0 + }, + "seaweedfs": { + "avg_cpu_percent": 1.6, + "peak_mem_mib": 546.3 + }, + "prometheus": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 74.3 + }, + "valkey": { + "avg_cpu_percent": 5.3, + "peak_mem_mib": 11.6 + } + }, + "collector_delta": { + "receiver_accepted_spans": 96287, + "receiver_accepted_log_records": 47752, + "receiver_accepted_metric_points": 372, + "receiver_refused_spans": 0, + "exporter_sent_spans": 96287, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 23527424, + "loki_bytes_delta": -54317056, + "errors_injected_500": 851, + "slow_requests_gt500ms": 853, + "traces_in_tempo": { + "all_http": 5000, + "error": 940, + "slow_gt500ms": 946 + }, + "kept_ratio": { + "errors": 1.105, + "slow": 1.109 + }, + "rps_runs": [ + 1311.6, + 1338.1 + ], + "p95_runs": [ + 74.2, + 70.25 + ], + "runs": [ + { + "profile": "on100", + "post": { + "requests": 40149, + "ok": 39347, + "errors": { + "500": 802 + }, + "seconds": 30, + "ok_rps": 1311.6, + "error_ratio": 0.02, + "p50_ms": 23.48, + "p95_ms": 74.2, + "p99_ms": 627.83 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.9, + "peak_mem_mib": 60.4 + }, + "api": { + "avg_cpu_percent": 83.8, + "peak_mem_mib": 86.0 + }, + "otel-collector": { + "avg_cpu_percent": 3.2, + "peak_mem_mib": 73.3 + }, + "tempo": { + "avg_cpu_percent": 5.1, + "peak_mem_mib": 437.9 + }, + "lb": { + "avg_cpu_percent": 21.7, + "peak_mem_mib": 24.1 + }, + "grafana": { + "avg_cpu_percent": 0.8, + "peak_mem_mib": 404.2 + }, + "rabbitmq": { + "avg_cpu_percent": 13.2, + "peak_mem_mib": 207.1 + }, + "postgres": { + "avg_cpu_percent": 97.0, + "peak_mem_mib": 372.8 + }, + "loki": { + "avg_cpu_percent": 3.9, + "peak_mem_mib": 477.8 + }, + "seaweedfs": { + "avg_cpu_percent": 1.5, + "peak_mem_mib": 546.3 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 74.3 + }, + "valkey": { + "avg_cpu_percent": 5.4, + "peak_mem_mib": 12.2 + } + }, + "collector_delta": { + "receiver_accepted_spans": 98971, + "receiver_accepted_log_records": 46175, + "receiver_accepted_metric_points": 341, + "receiver_refused_spans": 0, + "exporter_sent_spans": 93904, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": -24104960, + "loki_bytes_delta": 34271232, + "errors_injected_500": 818, + "slow_requests_gt500ms": 783, + "traces_in_tempo": { + "all_http": 5000, + "error": 904, + "slow_gt500ms": 893 + }, + "kept_ratio": { + "errors": 1.105, + "slow": 1.14 + } + }, + { + "profile": "on100", + "post": { + "requests": 40983, + "ok": 40144, + "errors": { + "500": 839 + }, + "seconds": 30, + "ok_rps": 1338.1, + "error_ratio": 0.0205, + "p50_ms": 22.75, + "p95_ms": 70.25, + "p99_ms": 628.03 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.8, + "peak_mem_mib": 59.7 + }, + "api": { + "avg_cpu_percent": 86.9, + "peak_mem_mib": 87.9 + }, + "otel-collector": { + "avg_cpu_percent": 3.1, + "peak_mem_mib": 70.8 + }, + "tempo": { + "avg_cpu_percent": 6.3, + "peak_mem_mib": 389.0 + }, + "lb": { + "avg_cpu_percent": 22.3, + "peak_mem_mib": 24.5 + }, + "grafana": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 410.1 + }, + "rabbitmq": { + "avg_cpu_percent": 8.5, + "peak_mem_mib": 205.7 + }, + "postgres": { + "avg_cpu_percent": 96.5, + "peak_mem_mib": 373.0 + }, + "loki": { + "avg_cpu_percent": 4.9, + "peak_mem_mib": 497.0 + }, + "seaweedfs": { + "avg_cpu_percent": 1.6, + "peak_mem_mib": 546.3 + }, + "prometheus": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 74.3 + }, + "valkey": { + "avg_cpu_percent": 5.3, + "peak_mem_mib": 11.6 + } + }, + "collector_delta": { + "receiver_accepted_spans": 96287, + "receiver_accepted_log_records": 47752, + "receiver_accepted_metric_points": 372, + "receiver_refused_spans": 0, + "exporter_sent_spans": 96287, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 23527424, + "loki_bytes_delta": -54317056, + "errors_injected_500": 851, + "slow_requests_gt500ms": 853, + "traces_in_tempo": { + "all_http": 5000, + "error": 940, + "slow_gt500ms": 946 + }, + "kept_ratio": { + "errors": 1.105, + "slow": 1.109 + } + } + ] + }, + { + "profile": "head10", + "post": { + "requests": 42159, + "ok": 41294, + "errors": { + "500": 865 + }, + "seconds": 30, + "ok_rps": 1376.5, + "error_ratio": 0.0205, + "p50_ms": 20.09, + "p95_ms": 70.82, + "p99_ms": 612.19 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 2.0, + "peak_mem_mib": 62.4 + }, + "api": { + "avg_cpu_percent": 76.8, + "peak_mem_mib": 87.0 + }, + "otel-collector": { + "avg_cpu_percent": 1.7, + "peak_mem_mib": 71.2 + }, + "tempo": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 301.0 + }, + "lb": { + "avg_cpu_percent": 21.7, + "peak_mem_mib": 24.3 + }, + "grafana": { + "avg_cpu_percent": 0.9, + "peak_mem_mib": 403.9 + }, + "rabbitmq": { + "avg_cpu_percent": 13.3, + "peak_mem_mib": 208.8 + }, + "postgres": { + "avg_cpu_percent": 98.0, + "peak_mem_mib": 372.3 + }, + "loki": { + "avg_cpu_percent": 4.0, + "peak_mem_mib": 465.0 + }, + "seaweedfs": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 546.3 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 76.9 + }, + "valkey": { + "avg_cpu_percent": 5.4, + "peak_mem_mib": 11.4 + } + }, + "collector_delta": { + "receiver_accepted_spans": 9927, + "receiver_accepted_log_records": 49103, + "receiver_accepted_metric_points": 372, + "receiver_refused_spans": 0, + "exporter_sent_spans": 10422, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": -5586944, + "loki_bytes_delta": -53788672, + "errors_injected_500": 872, + "slow_requests_gt500ms": 834, + "traces_in_tempo": { + "all_http": 4904, + "error": 113, + "slow_gt500ms": 77 + }, + "kept_ratio": { + "errors": 0.13, + "slow": 0.092 + }, + "rps_runs": [ + 1424.9, + 1376.5 + ], + "p95_runs": [ + 64.1, + 70.82 + ], + "runs": [ + { + "profile": "head10", + "post": { + "requests": 43671, + "ok": 42747, + "errors": { + "500": 924 + }, + "seconds": 30, + "ok_rps": 1424.9, + "error_ratio": 0.0212, + "p50_ms": 18.5, + "p95_ms": 64.1, + "p99_ms": 621.33 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.5, + "peak_mem_mib": 60.3 + }, + "api": { + "avg_cpu_percent": 74.1, + "peak_mem_mib": 84.0 + }, + "otel-collector": { + "avg_cpu_percent": 2.0, + "peak_mem_mib": 70.9 + }, + "tempo": { + "avg_cpu_percent": 2.2, + "peak_mem_mib": 260.5 + }, + "lb": { + "avg_cpu_percent": 22.2, + "peak_mem_mib": 24.0 + }, + "grafana": { + "avg_cpu_percent": 1.1, + "peak_mem_mib": 405.1 + }, + "rabbitmq": { + "avg_cpu_percent": 10.9, + "peak_mem_mib": 196.5 + }, + "postgres": { + "avg_cpu_percent": 99.3, + "peak_mem_mib": 373.8 + }, + "loki": { + "avg_cpu_percent": 4.4, + "peak_mem_mib": 500.8 + }, + "seaweedfs": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 546.3 + }, + "prometheus": { + "avg_cpu_percent": 0.4, + "peak_mem_mib": 76.6 + }, + "valkey": { + "avg_cpu_percent": 5.3, + "peak_mem_mib": 11.5 + } + }, + "collector_delta": { + "receiver_accepted_spans": 11172, + "receiver_accepted_log_records": 50129, + "receiver_accepted_metric_points": 341, + "receiver_refused_spans": 0, + "exporter_sent_spans": 10205, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 2437120, + "loki_bytes_delta": 43749376, + "errors_injected_500": 935, + "slow_requests_gt500ms": 895, + "traces_in_tempo": { + "all_http": 4935, + "error": 108, + "slow_gt500ms": 86 + }, + "kept_ratio": { + "errors": 0.116, + "slow": 0.096 + } + }, + { + "profile": "head10", + "post": { + "requests": 42159, + "ok": 41294, + "errors": { + "500": 865 + }, + "seconds": 30, + "ok_rps": 1376.5, + "error_ratio": 0.0205, + "p50_ms": 20.09, + "p95_ms": 70.82, + "p99_ms": 612.19 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 2.0, + "peak_mem_mib": 62.4 + }, + "api": { + "avg_cpu_percent": 76.8, + "peak_mem_mib": 87.0 + }, + "otel-collector": { + "avg_cpu_percent": 1.7, + "peak_mem_mib": 71.2 + }, + "tempo": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 301.0 + }, + "lb": { + "avg_cpu_percent": 21.7, + "peak_mem_mib": 24.3 + }, + "grafana": { + "avg_cpu_percent": 0.9, + "peak_mem_mib": 403.9 + }, + "rabbitmq": { + "avg_cpu_percent": 13.3, + "peak_mem_mib": 208.8 + }, + "postgres": { + "avg_cpu_percent": 98.0, + "peak_mem_mib": 372.3 + }, + "loki": { + "avg_cpu_percent": 4.0, + "peak_mem_mib": 465.0 + }, + "seaweedfs": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 546.3 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 76.9 + }, + "valkey": { + "avg_cpu_percent": 5.4, + "peak_mem_mib": 11.4 + } + }, + "collector_delta": { + "receiver_accepted_spans": 9927, + "receiver_accepted_log_records": 49103, + "receiver_accepted_metric_points": 372, + "receiver_refused_spans": 0, + "exporter_sent_spans": 10422, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": -5586944, + "loki_bytes_delta": -53788672, + "errors_injected_500": 872, + "slow_requests_gt500ms": 834, + "traces_in_tempo": { + "all_http": 4904, + "error": 113, + "slow_gt500ms": 77 + }, + "kept_ratio": { + "errors": 0.13, + "slow": 0.092 + } + } + ] + }, + { + "profile": "tail", + "post": { + "requests": 42956, + "ok": 42071, + "errors": { + "500": 885 + }, + "seconds": 30, + "ok_rps": 1402.4, + "error_ratio": 0.0206, + "p50_ms": 20.69, + "p95_ms": 63.76, + "p99_ms": 625.38 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.9, + "peak_mem_mib": 59.7 + }, + "api": { + "avg_cpu_percent": 84.9, + "peak_mem_mib": 86.5 + }, + "otel-collector": { + "avg_cpu_percent": 4.0, + "peak_mem_mib": 110.5 + }, + "tempo": { + "avg_cpu_percent": 1.2, + "peak_mem_mib": 311.2 + }, + "lb": { + "avg_cpu_percent": 21.1, + "peak_mem_mib": 24.1 + }, + "grafana": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 404.0 + }, + "rabbitmq": { + "avg_cpu_percent": 9.7, + "peak_mem_mib": 174.7 + }, + "postgres": { + "avg_cpu_percent": 96.8, + "peak_mem_mib": 375.0 + }, + "loki": { + "avg_cpu_percent": 5.0, + "peak_mem_mib": 471.8 + }, + "seaweedfs": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 546.4 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 78.8 + }, + "valkey": { + "avg_cpu_percent": 5.2, + "peak_mem_mib": 11.8 + } + }, + "collector_delta": { + "receiver_accepted_spans": 101134, + "receiver_accepted_log_records": 49659, + "receiver_accepted_metric_points": 372, + "receiver_refused_spans": 0, + "exporter_sent_spans": 14954, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": -4780032, + "loki_bytes_delta": 38547456, + "errors_injected_500": 899, + "slow_requests_gt500ms": 944, + "traces_in_tempo": { + "all_http": 5000, + "error": 1003, + "slow_gt500ms": 1054 + }, + "kept_ratio": { + "errors": 1.116, + "slow": 1.117 + }, + "rps_runs": [ + 1396.8, + 1402.4 + ], + "p95_runs": [ + 70.29, + 63.76 + ], + "runs": [ + { + "profile": "tail", + "post": { + "requests": 42797, + "ok": 41904, + "errors": { + "500": 893 + }, + "seconds": 30, + "ok_rps": 1396.8, + "error_ratio": 0.0209, + "p50_ms": 20.92, + "p95_ms": 70.29, + "p99_ms": 625.38 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.8, + "peak_mem_mib": 61.7 + }, + "api": { + "avg_cpu_percent": 84.4, + "peak_mem_mib": 87.3 + }, + "otel-collector": { + "avg_cpu_percent": 3.9, + "peak_mem_mib": 110.4 + }, + "tempo": { + "avg_cpu_percent": 2.4, + "peak_mem_mib": 330.1 + }, + "lb": { + "avg_cpu_percent": 21.8, + "peak_mem_mib": 24.6 + }, + "grafana": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 403.9 + }, + "rabbitmq": { + "avg_cpu_percent": 10.3, + "peak_mem_mib": 194.5 + }, + "postgres": { + "avg_cpu_percent": 97.1, + "peak_mem_mib": 373.2 + }, + "loki": { + "avg_cpu_percent": 4.5, + "peak_mem_mib": 460.9 + }, + "seaweedfs": { + "avg_cpu_percent": 1.5, + "peak_mem_mib": 546.4 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 76.7 + }, + "valkey": { + "avg_cpu_percent": 5.3, + "peak_mem_mib": 14.1 + } + }, + "collector_delta": { + "receiver_accepted_spans": 99880, + "receiver_accepted_log_records": 49202, + "receiver_accepted_metric_points": 341, + "receiver_refused_spans": 0, + "exporter_sent_spans": 14046, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": -1884160, + "loki_bytes_delta": 33947648, + "errors_injected_500": 908, + "slow_requests_gt500ms": 875, + "traces_in_tempo": { + "all_http": 5000, + "error": 1013, + "slow_gt500ms": 985 + }, + "kept_ratio": { + "errors": 1.116, + "slow": 1.126 + } + }, + { + "profile": "tail", + "post": { + "requests": 42956, + "ok": 42071, + "errors": { + "500": 885 + }, + "seconds": 30, + "ok_rps": 1402.4, + "error_ratio": 0.0206, + "p50_ms": 20.69, + "p95_ms": 63.76, + "p99_ms": 625.38 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.9, + "peak_mem_mib": 59.7 + }, + "api": { + "avg_cpu_percent": 84.9, + "peak_mem_mib": 86.5 + }, + "otel-collector": { + "avg_cpu_percent": 4.0, + "peak_mem_mib": 110.5 + }, + "tempo": { + "avg_cpu_percent": 1.2, + "peak_mem_mib": 311.2 + }, + "lb": { + "avg_cpu_percent": 21.1, + "peak_mem_mib": 24.1 + }, + "grafana": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 404.0 + }, + "rabbitmq": { + "avg_cpu_percent": 9.7, + "peak_mem_mib": 174.7 + }, + "postgres": { + "avg_cpu_percent": 96.8, + "peak_mem_mib": 375.0 + }, + "loki": { + "avg_cpu_percent": 5.0, + "peak_mem_mib": 471.8 + }, + "seaweedfs": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 546.4 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 78.8 + }, + "valkey": { + "avg_cpu_percent": 5.2, + "peak_mem_mib": 11.8 + } + }, + "collector_delta": { + "receiver_accepted_spans": 101134, + "receiver_accepted_log_records": 49659, + "receiver_accepted_metric_points": 372, + "receiver_refused_spans": 0, + "exporter_sent_spans": 14954, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": -4780032, + "loki_bytes_delta": 38547456, + "errors_injected_500": 899, + "slow_requests_gt500ms": 944, + "traces_in_tempo": { + "all_http": 5000, + "error": 1003, + "slow_gt500ms": 1054 + }, + "kept_ratio": { + "errors": 1.116, + "slow": 1.117 + } + } + ] + }, + { + "profile": "on100_bigq", + "post": { + "requests": 41207, + "ok": 40393, + "errors": { + "500": 814 + }, + "seconds": 30, + "ok_rps": 1346.4, + "error_ratio": 0.0198, + "p50_ms": 20.68, + "p95_ms": 76.9, + "p99_ms": 625.55 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.7, + "peak_mem_mib": 62.2 + }, + "api": { + "avg_cpu_percent": 83.9, + "peak_mem_mib": 89.8 + }, + "otel-collector": { + "avg_cpu_percent": 2.9, + "peak_mem_mib": 72.2 + }, + "tempo": { + "avg_cpu_percent": 3.1, + "peak_mem_mib": 387.2 + }, + "lb": { + "avg_cpu_percent": 21.6, + "peak_mem_mib": 24.4 + }, + "grafana": { + "avg_cpu_percent": 1.1, + "peak_mem_mib": 403.9 + }, + "rabbitmq": { + "avg_cpu_percent": 14.3, + "peak_mem_mib": 211.9 + }, + "postgres": { + "avg_cpu_percent": 96.8, + "peak_mem_mib": 368.8 + }, + "loki": { + "avg_cpu_percent": 4.3, + "peak_mem_mib": 482.2 + }, + "seaweedfs": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 546.3 + }, + "prometheus": { + "avg_cpu_percent": 0.4, + "peak_mem_mib": 80.9 + }, + "valkey": { + "avg_cpu_percent": 5.5, + "peak_mem_mib": 11.7 + } + }, + "collector_delta": { + "receiver_accepted_spans": 96033, + "receiver_accepted_log_records": 47183, + "receiver_accepted_metric_points": 372, + "receiver_refused_spans": 0, + "exporter_sent_spans": 96033, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": -18698240, + "loki_bytes_delta": -87719936, + "errors_injected_500": 826, + "slow_requests_gt500ms": 840, + "traces_in_tempo": { + "all_http": 5000, + "error": 905, + "slow_gt500ms": 950 + }, + "kept_ratio": { + "errors": 1.096, + "slow": 1.131 + }, + "rps_runs": [ + 1423.1, + 1346.4 + ], + "p95_runs": [ + 73.18, + 76.9 + ], + "runs": [ + { + "profile": "on100_bigq", + "post": { + "requests": 43580, + "ok": 42693, + "errors": { + "500": 887 + }, + "seconds": 30, + "ok_rps": 1423.1, + "error_ratio": 0.0204, + "p50_ms": 18.41, + "p95_ms": 73.18, + "p99_ms": 622.45 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 2.2, + "peak_mem_mib": 62.2 + }, + "api": { + "avg_cpu_percent": 80.6, + "peak_mem_mib": 80.2 + }, + "otel-collector": { + "avg_cpu_percent": 3.1, + "peak_mem_mib": 71.6 + }, + "tempo": { + "avg_cpu_percent": 3.5, + "peak_mem_mib": 381.4 + }, + "lb": { + "avg_cpu_percent": 21.1, + "peak_mem_mib": 24.3 + }, + "grafana": { + "avg_cpu_percent": 1.0, + "peak_mem_mib": 403.9 + }, + "rabbitmq": { + "avg_cpu_percent": 12.5, + "peak_mem_mib": 200.6 + }, + "postgres": { + "avg_cpu_percent": 97.4, + "peak_mem_mib": 379.6 + }, + "loki": { + "avg_cpu_percent": 5.2, + "peak_mem_mib": 479.5 + }, + "seaweedfs": { + "avg_cpu_percent": 1.2, + "peak_mem_mib": 546.3 + }, + "prometheus": { + "avg_cpu_percent": 0.3, + "peak_mem_mib": 78.9 + }, + "valkey": { + "avg_cpu_percent": 5.1, + "peak_mem_mib": 11.4 + } + }, + "collector_delta": { + "receiver_accepted_spans": 108579, + "receiver_accepted_log_records": 52295, + "receiver_accepted_metric_points": 372, + "receiver_refused_spans": 0, + "exporter_sent_spans": 101589, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 22175744, + "loki_bytes_delta": 39370752, + "errors_injected_500": 900, + "slow_requests_gt500ms": 912, + "traces_in_tempo": { + "all_http": 5000, + "error": 996, + "slow_gt500ms": 1020 + }, + "kept_ratio": { + "errors": 1.107, + "slow": 1.118 + } + }, + { + "profile": "on100_bigq", + "post": { + "requests": 41207, + "ok": 40393, + "errors": { + "500": 814 + }, + "seconds": 30, + "ok_rps": 1346.4, + "error_ratio": 0.0198, + "p50_ms": 20.68, + "p95_ms": 76.9, + "p99_ms": 625.55 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.7, + "peak_mem_mib": 62.2 + }, + "api": { + "avg_cpu_percent": 83.9, + "peak_mem_mib": 89.8 + }, + "otel-collector": { + "avg_cpu_percent": 2.9, + "peak_mem_mib": 72.2 + }, + "tempo": { + "avg_cpu_percent": 3.1, + "peak_mem_mib": 387.2 + }, + "lb": { + "avg_cpu_percent": 21.6, + "peak_mem_mib": 24.4 + }, + "grafana": { + "avg_cpu_percent": 1.1, + "peak_mem_mib": 403.9 + }, + "rabbitmq": { + "avg_cpu_percent": 14.3, + "peak_mem_mib": 211.9 + }, + "postgres": { + "avg_cpu_percent": 96.8, + "peak_mem_mib": 368.8 + }, + "loki": { + "avg_cpu_percent": 4.3, + "peak_mem_mib": 482.2 + }, + "seaweedfs": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 546.3 + }, + "prometheus": { + "avg_cpu_percent": 0.4, + "peak_mem_mib": 80.9 + }, + "valkey": { + "avg_cpu_percent": 5.5, + "peak_mem_mib": 11.7 + } + }, + "collector_delta": { + "receiver_accepted_spans": 96033, + "receiver_accepted_log_records": 47183, + "receiver_accepted_metric_points": 372, + "receiver_refused_spans": 0, + "exporter_sent_spans": 96033, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": -18698240, + "loki_bytes_delta": -87719936, + "errors_injected_500": 826, + "slow_requests_gt500ms": 840, + "traces_in_tempo": { + "all_http": 5000, + "error": 905, + "slow_gt500ms": 950 + }, + "kept_ratio": { + "errors": 1.096, + "slow": 1.131 + } + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/obs_cost.png b/evidence/obs_cost.png new file mode 100644 index 0000000..dc3f50b Binary files /dev/null and b/evidence/obs_cost.png differ diff --git a/evidence/recheck_2026-11-12/clean-ports.yaml b/evidence/recheck_2026-11-12/clean-ports.yaml new file mode 100644 index 0000000..3192394 --- /dev/null +++ b/evidence/recheck_2026-11-12/clean-ports.yaml @@ -0,0 +1,5 @@ +services: + lb: + ports: !override ["127.0.0.1:18080:8080"] + grafana: + ports: !override ["127.0.0.1:13000:3000"] diff --git a/evidence/recheck_2026-11-12/clean_clone.txt b/evidence/recheck_2026-11-12/clean_clone.txt new file mode 100644 index 0000000..20c32be --- /dev/null +++ b/evidence/recheck_2026-11-12/clean_clone.txt @@ -0,0 +1,345 @@ +$ git rev-parse HEAD +6d87f5426c555ea996d97f1415f50cd3bf7b1f17 +exit=0 + +$ git status --short + +exit=0 + +$ C:/Program Files/Git/bin/bash.exe --noprofile --norc -c sh scripts/init-secrets.sh + +exit=0 + +$ docker compose -p mtusi-lab6-clean-check -f compose.yaml -f C:\Users\kolya5544\source\repos\vmdMTUCI\tools\lab6-qa\clean-ports.yaml config --quiet + +exit=0 + +$ docker compose -p mtusi-lab6-clean-check -f compose.yaml -f C:\Users\kolya5544\source\repos\vmdMTUCI\tools\lab6-qa\clean-ports.yaml up -d --build --wait --wait-timeout 180 + Image mtusi-messenger:lab6 Building + Image mtusi-messenger:lab6 Building + Image mtusi-messenger:lab6 Building + Image mtusi-messenger:lab6 Building +#1 [internal] load local bake definitions +#1 reading from stdin 2.19kB 0.0s done +#1 DONE 0.0s + +#2 [worker internal] load build definition from Dockerfile +#2 transferring dockerfile: 1.91kB done +#2 DONE 0.1s + +#2 [s3-init internal] load build definition from Dockerfile +#2 DONE 0.1s + +#3 [s3-init] resolve image config for docker-image://docker.io/docker/dockerfile:1 +#3 DONE 3.9s + +#4 [migrate] docker-image://docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e +#4 resolve docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e 0.1s done +#4 CACHED + +#5 [s3-init internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c +#5 DONE 1.9s + +#6 [s3-init internal] load .dockerignore +#6 transferring context: 289B done +#6 DONE 0.1s + +#7 [api builder 1/4] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c +#7 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c 0.1s done +#7 DONE 0.1s + +#8 [api internal] load build context +#8 transferring context: 159.42kB 0.0s done +#8 DONE 0.2s + +#9 [migrate builder 4/4] RUN python -m venv /opt/venv && /opt/venv/bin/pip install -r requirements.txt +#9 CACHED + +#10 [migrate stage-1 3/6] COPY --from=builder /opt/venv /opt/venv +#10 CACHED + +#11 [migrate stage-1 2/6] RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home app +#11 CACHED + +#12 [migrate builder 2/4] WORKDIR /build +#12 CACHED + +#13 [migrate builder 3/4] COPY requirements.txt . +#13 CACHED + +#14 [s3-init stage-1 4/6] WORKDIR /srv +#14 CACHED + +#15 [api stage-1 5/6] COPY src/ /srv/src/ +#15 DONE 0.1s + +#15 [worker stage-1 5/6] COPY src/ /srv/src/ +#15 DONE 0.1s + +#16 [api stage-1 6/6] COPY migrations/ /srv/migrations/ +#16 DONE 0.3s + +#16 [worker stage-1 6/6] COPY migrations/ /srv/migrations/ +#16 DONE 0.3s + +#17 [api] exporting to image +#17 exporting layers +#17 exporting layers 0.4s done +#17 exporting manifest sha256:a44a3db4f954b2d839b7fe29cbb71c7f7674f8ca74a7fa09e656d6ca54c5764c +#17 exporting manifest sha256:a44a3db4f954b2d839b7fe29cbb71c7f7674f8ca74a7fa09e656d6ca54c5764c 0.3s done +#17 exporting config sha256:7d05c42b9fa6c5dde578b5fe27fcd0b08b41833510fe211417d886af50eff697 +#17 exporting config sha256:7d05c42b9fa6c5dde578b5fe27fcd0b08b41833510fe211417d886af50eff697 0.2s done +#17 exporting attestation manifest sha256:b20893808f6cfa06f171dbcbc4bea0298f024d1011fec1f869b4c2ec35796cf6 +#17 exporting attestation manifest sha256:b20893808f6cfa06f171dbcbc4bea0298f024d1011fec1f869b4c2ec35796cf6 0.5s done +#17 exporting manifest list sha256:bfc4140d1931f7f6fc6ab531b42b5cafbacdc8bd5f242c7ba7048342ae0d792f +#17 exporting manifest list sha256:bfc4140d1931f7f6fc6ab531b42b5cafbacdc8bd5f242c7ba7048342ae0d792f 0.2s done +#17 naming to docker.io/library/mtusi-messenger:lab6 0.0s done +#17 unpacking to docker.io/library/mtusi-messenger:lab6 +#17 unpacking to docker.io/library/mtusi-messenger:lab6 0.6s done +#17 DONE 2.5s + +#18 [migrate] exporting to image +#18 exporting layers 0.4s done +#18 exporting manifest sha256:5308c23deb5b344ce251b0b2bca861d6bd2bc58fc324fb5da4edd2fa3987922c 0.2s done +#18 exporting config sha256:44c24ac50125b68dca2d8cf40743ae2bc78fd95835a54519d19de53dbc8458da 0.2s done +#18 exporting attestation manifest sha256:6ca67fd654b1854f407e84829e0363921d0ee4749e7072f7524c8547426e4ec4 0.5s done +#18 exporting manifest list sha256:f9ccfe31b60f1164f9f0c5a6502e502b785d096ebe59aaa7502191d8eaae6407 0.2s done +#18 naming to docker.io/library/mtusi-messenger:lab6 0.1s done +#18 unpacking to docker.io/library/mtusi-messenger:lab6 0.4s done +#18 DONE 2.4s + +#19 [s3-init] exporting to image +#19 exporting layers 0.5s done +#19 exporting manifest sha256:a1a52ab91fb42571d42357271ff4f44839686f1e9a9e5cab34e3fd45ddaabe77 0.1s done +#19 exporting config sha256:469a339ae5debdb4c78f89c63756e83b237176eda4b84c070a4c3b4bc7206f76 0.2s done +#19 exporting attestation manifest sha256:9849897140fe8d411579f10ec3b967c386258d69a347d282774217cb3ea5d4e4 0.5s done +#19 exporting manifest list sha256:e0ecdb5c24bc281b6861cbb6a7861794e8b9692f22ac3383b9725a2a68c385de 0.2s done +#19 naming to docker.io/library/mtusi-messenger:lab6 0.1s done +#19 unpacking to docker.io/library/mtusi-messenger:lab6 0.5s done +#19 DONE 2.4s + +#20 [worker] exporting to image +#20 exporting layers 0.5s done +#20 exporting manifest sha256:9dccceba30cebf826f796751a00139cadc366f85fa6575ca7e5dbeaaa07ac69f 0.2s done +#20 exporting config sha256:fe76586a0b246816ab5ea92a6e2f7040c19c8132a975a8a4d76a956ac08b6b85 0.2s done +#20 exporting attestation manifest sha256:5c98251a245b9dda793e1c6841b23918675605ae0962a14e0b6699f66bcb7bec 0.5s done +#20 exporting manifest list sha256:7f326220eb027a78f09705d6f233b029bd58f56e6da48d22c8acaad5102d781c 0.3s done +#20 naming to docker.io/library/mtusi-messenger:lab6 0.1s done +#20 unpacking to docker.io/library/mtusi-messenger:lab6 0.6s done +#20 DONE 2.5s + +#21 [migrate] resolving provenance for metadata file +#21 DONE 0.4s + +#22 [worker] resolving provenance for metadata file +#22 DONE 0.2s + +#23 [s3-init] resolving provenance for metadata file +#23 DONE 0.4s + +#24 [api] resolving provenance for metadata file +#24 DONE 0.0s + Image mtusi-messenger:lab6 Built + Image mtusi-messenger:lab6 Built + Image mtusi-messenger:lab6 Built + Image mtusi-messenger:lab6 Built + Network mtusi-lab6-clean-check_backend Creating + Network mtusi-lab6-clean-check_backend Created + Volume mtusi-lab6-clean-check_rabbitmqdata Creating + Volume mtusi-lab6-clean-check_rabbitmqdata Created + Volume mtusi-lab6-clean-check_valkeydata Creating + Volume mtusi-lab6-clean-check_valkeydata Created + Volume mtusi-lab6-clean-check_pgdata Creating + Volume mtusi-lab6-clean-check_pgdata Created + Volume mtusi-lab6-clean-check_lokidata Creating + Volume mtusi-lab6-clean-check_lokidata Created + Volume mtusi-lab6-clean-check_seaweeddata Creating + Volume mtusi-lab6-clean-check_seaweeddata Created + Volume mtusi-lab6-clean-check_promdata Creating + Volume mtusi-lab6-clean-check_promdata Created + Volume mtusi-lab6-clean-check_tempodata Creating + Volume mtusi-lab6-clean-check_tempodata Created + Volume mtusi-lab6-clean-check_grafanadata Creating + Volume mtusi-lab6-clean-check_grafanadata Created + Container mtusi-lab6-clean-check-valkey-1 Creating + Container mtusi-lab6-clean-check-rabbitmq-1 Creating + Container mtusi-lab6-clean-check-otel-collector-1 Creating + Container mtusi-lab6-clean-check-postgres-1 Creating + Container mtusi-lab6-clean-check-seaweedfs-1 Creating + Container mtusi-lab6-clean-check-tempo-1 Creating + Container mtusi-lab6-clean-check-prometheus-1 Creating + Container mtusi-lab6-clean-check-loki-1 Creating +time="2026-11-12T04:51:24+03:00" level=warning msg="secrets `uid`, `gid` and `mode` are not supported, they will be ignored" + Container mtusi-lab6-clean-check-valkey-1 Created + Container mtusi-lab6-clean-check-prometheus-1 Created + Container mtusi-lab6-clean-check-tempo-1 Created + Container mtusi-lab6-clean-check-rabbitmq-1 Created + Container mtusi-lab6-clean-check-loki-1 Created + Container mtusi-lab6-clean-check-grafana-1 Creating + Container mtusi-lab6-clean-check-otel-collector-1 Created + Container mtusi-lab6-clean-check-postgres-1 Created + Container mtusi-lab6-clean-check-migrate-1 Creating + Container mtusi-lab6-clean-check-seaweedfs-1 Created + Container mtusi-lab6-clean-check-s3-init-1 Creating + Container mtusi-lab6-clean-check-grafana-1 Created + Container mtusi-lab6-clean-check-migrate-1 Created + Container mtusi-lab6-clean-check-s3-init-1 Created + Container mtusi-lab6-clean-check-worker-1 Creating + Container mtusi-lab6-clean-check-api-2 Creating + Container mtusi-lab6-clean-check-api-1 Creating + Container mtusi-lab6-clean-check-api-2 Created + Container mtusi-lab6-clean-check-worker-1 Created + Container mtusi-lab6-clean-check-api-1 Created + Container mtusi-lab6-clean-check-lb-1 Creating + Container mtusi-lab6-clean-check-lb-1 Created + Container mtusi-lab6-clean-check-seaweedfs-1 Starting + Container mtusi-lab6-clean-check-loki-1 Starting + Container mtusi-lab6-clean-check-tempo-1 Starting + Container mtusi-lab6-clean-check-valkey-1 Starting + Container mtusi-lab6-clean-check-rabbitmq-1 Starting + Container mtusi-lab6-clean-check-postgres-1 Starting + Container mtusi-lab6-clean-check-prometheus-1 Starting + Container mtusi-lab6-clean-check-otel-collector-1 Starting + Container mtusi-lab6-clean-check-loki-1 Started + Container mtusi-lab6-clean-check-seaweedfs-1 Started + Container mtusi-lab6-clean-check-seaweedfs-1 Waiting + Container mtusi-lab6-clean-check-rabbitmq-1 Started + Container mtusi-lab6-clean-check-valkey-1 Started + Container mtusi-lab6-clean-check-tempo-1 Started + Container mtusi-lab6-clean-check-postgres-1 Started + Container mtusi-lab6-clean-check-postgres-1 Waiting + Container mtusi-lab6-clean-check-otel-collector-1 Started + Container mtusi-lab6-clean-check-prometheus-1 Started + Container mtusi-lab6-clean-check-grafana-1 Starting + Container mtusi-lab6-clean-check-grafana-1 Started + Container mtusi-lab6-clean-check-seaweedfs-1 Healthy + Container mtusi-lab6-clean-check-s3-init-1 Starting + Container mtusi-lab6-clean-check-s3-init-1 Started + Container mtusi-lab6-clean-check-postgres-1 Healthy + Container mtusi-lab6-clean-check-migrate-1 Starting + Container mtusi-lab6-clean-check-migrate-1 Started + Container mtusi-lab6-clean-check-s3-init-1 Waiting + Container mtusi-lab6-clean-check-valkey-1 Waiting + Container mtusi-lab6-clean-check-rabbitmq-1 Waiting + Container mtusi-lab6-clean-check-migrate-1 Waiting + Container mtusi-lab6-clean-check-migrate-1 Waiting + Container mtusi-lab6-clean-check-s3-init-1 Waiting + Container mtusi-lab6-clean-check-s3-init-1 Exited + Container mtusi-lab6-clean-check-s3-init-1 Exited + Container mtusi-lab6-clean-check-rabbitmq-1 Healthy + Container mtusi-lab6-clean-check-valkey-1 Healthy + Container mtusi-lab6-clean-check-migrate-1 Exited + Container mtusi-lab6-clean-check-worker-1 Starting + Container mtusi-lab6-clean-check-migrate-1 Exited + Container mtusi-lab6-clean-check-api-1 Starting + Container mtusi-lab6-clean-check-api-1 Started + Container mtusi-lab6-clean-check-api-2 Starting + Container mtusi-lab6-clean-check-worker-1 Started + Container mtusi-lab6-clean-check-api-2 Started + Container mtusi-lab6-clean-check-lb-1 Starting + Container mtusi-lab6-clean-check-lb-1 Started + Container mtusi-lab6-clean-check-grafana-1 Waiting + Container mtusi-lab6-clean-check-valkey-1 Waiting + Container mtusi-lab6-clean-check-api-1 Waiting + Container mtusi-lab6-clean-check-api-2 Waiting + Container mtusi-lab6-clean-check-postgres-1 Waiting + Container mtusi-lab6-clean-check-loki-1 Waiting + Container mtusi-lab6-clean-check-migrate-1 Waiting + Container mtusi-lab6-clean-check-otel-collector-1 Waiting + Container mtusi-lab6-clean-check-tempo-1 Waiting + Container mtusi-lab6-clean-check-worker-1 Waiting + Container mtusi-lab6-clean-check-s3-init-1 Waiting + Container mtusi-lab6-clean-check-rabbitmq-1 Waiting + Container mtusi-lab6-clean-check-seaweedfs-1 Waiting + Container mtusi-lab6-clean-check-prometheus-1 Waiting + Container mtusi-lab6-clean-check-lb-1 Waiting + Container mtusi-lab6-clean-check-otel-collector-1 Healthy + Container mtusi-lab6-clean-check-rabbitmq-1 Healthy + Container mtusi-lab6-clean-check-seaweedfs-1 Healthy + Container mtusi-lab6-clean-check-postgres-1 Healthy + Container mtusi-lab6-clean-check-tempo-1 Healthy + Container mtusi-lab6-clean-check-prometheus-1 Healthy + Container mtusi-lab6-clean-check-loki-1 Healthy + Container mtusi-lab6-clean-check-valkey-1 Healthy + Container mtusi-lab6-clean-check-migrate-1 Exited + Container mtusi-lab6-clean-check-s3-init-1 Exited + Container mtusi-lab6-clean-check-worker-1 Healthy + Container mtusi-lab6-clean-check-api-1 Healthy + Container mtusi-lab6-clean-check-api-2 Healthy + Container mtusi-lab6-clean-check-lb-1 Healthy + Container mtusi-lab6-clean-check-grafana-1 Healthy +exit=0 + +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +exit=0 + +$ python tests/smoke.py --base-url http://127.0.0.1:18080 +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +exit=0 + +PASS: fresh secrets/volumes, public API, 3 datasources, provisioned dashboard with 22 panels +$ docker compose -p mtusi-lab6-clean-check -f compose.yaml -f C:\Users\kolya5544\source\repos\vmdMTUCI\tools\lab6-qa\clean-ports.yaml down + Container mtusi-lab6-clean-check-otel-collector-1 Stopping + Container mtusi-lab6-clean-check-lb-1 Stopping + Container mtusi-lab6-clean-check-grafana-1 Stopping + Container mtusi-lab6-clean-check-worker-1 Stopping + Container mtusi-lab6-clean-check-otel-collector-1 Stopped + Container mtusi-lab6-clean-check-otel-collector-1 Removing + Container mtusi-lab6-clean-check-otel-collector-1 Removed + Container mtusi-lab6-clean-check-lb-1 Stopped + Container mtusi-lab6-clean-check-lb-1 Removing + Container mtusi-lab6-clean-check-lb-1 Removed + Container mtusi-lab6-clean-check-api-2 Stopping + Container mtusi-lab6-clean-check-api-1 Stopping + Container mtusi-lab6-clean-check-grafana-1 Stopped + Container mtusi-lab6-clean-check-grafana-1 Removing + Container mtusi-lab6-clean-check-grafana-1 Removed + Container mtusi-lab6-clean-check-loki-1 Stopping + Container mtusi-lab6-clean-check-tempo-1 Stopping + Container mtusi-lab6-clean-check-prometheus-1 Stopping + Container mtusi-lab6-clean-check-prometheus-1 Stopped + Container mtusi-lab6-clean-check-prometheus-1 Removing + Container mtusi-lab6-clean-check-prometheus-1 Removed + Container mtusi-lab6-clean-check-tempo-1 Stopped + Container mtusi-lab6-clean-check-tempo-1 Removing + Container mtusi-lab6-clean-check-tempo-1 Removed + Container mtusi-lab6-clean-check-loki-1 Stopped + Container mtusi-lab6-clean-check-loki-1 Removing + Container mtusi-lab6-clean-check-loki-1 Removed + Container mtusi-lab6-clean-check-worker-1 Stopped + Container mtusi-lab6-clean-check-worker-1 Removing + Container mtusi-lab6-clean-check-worker-1 Removed + Container mtusi-lab6-clean-check-rabbitmq-1 Stopping + Container mtusi-lab6-clean-check-api-2 Stopped + Container mtusi-lab6-clean-check-api-2 Removing + Container mtusi-lab6-clean-check-api-2 Removed + Container mtusi-lab6-clean-check-rabbitmq-1 Stopped + Container mtusi-lab6-clean-check-rabbitmq-1 Removing + Container mtusi-lab6-clean-check-rabbitmq-1 Removed + Container mtusi-lab6-clean-check-api-1 Stopped + Container mtusi-lab6-clean-check-api-1 Removing + Container mtusi-lab6-clean-check-api-1 Removed + Container mtusi-lab6-clean-check-migrate-1 Stopping + Container mtusi-lab6-clean-check-valkey-1 Stopping + Container mtusi-lab6-clean-check-s3-init-1 Stopping + Container mtusi-lab6-clean-check-migrate-1 Stopped + Container mtusi-lab6-clean-check-migrate-1 Removing + Container mtusi-lab6-clean-check-s3-init-1 Stopped + Container mtusi-lab6-clean-check-s3-init-1 Removing + Container mtusi-lab6-clean-check-s3-init-1 Removed + Container mtusi-lab6-clean-check-seaweedfs-1 Stopping + Container mtusi-lab6-clean-check-migrate-1 Removed + Container mtusi-lab6-clean-check-postgres-1 Stopping + Container mtusi-lab6-clean-check-valkey-1 Stopped + Container mtusi-lab6-clean-check-valkey-1 Removing + Container mtusi-lab6-clean-check-postgres-1 Stopped + Container mtusi-lab6-clean-check-postgres-1 Removing + Container mtusi-lab6-clean-check-valkey-1 Removed + Container mtusi-lab6-clean-check-postgres-1 Removed + Container mtusi-lab6-clean-check-seaweedfs-1 Stopped + Container mtusi-lab6-clean-check-seaweedfs-1 Removing + Container mtusi-lab6-clean-check-seaweedfs-1 Removed + Network mtusi-lab6-clean-check_backend Removing + Network mtusi-lab6-clean-check_backend Removed +exit=0 diff --git a/evidence/recheck_2026-11-12/commands.txt b/evidence/recheck_2026-11-12/commands.txt new file mode 100644 index 0000000..96e380c --- /dev/null +++ b/evidence/recheck_2026-11-12/commands.txt @@ -0,0 +1,136 @@ +$ python -u research/incidents.py --out evidence\recheck_2026-11-12 +{ + "incidents": [ + { + "incident": "worker_down", + "alert": "QueueLagGrowing", + "injected_at_s": 25.0, + "firing_at_s": 93.3, + "detection_seconds": 68.3, + "recovered_cmd_at_s": 96.5, + "resolved_at_s": 118.3, + "fired": true, + "resolved": true, + "evidence": { + "queue_oldest_age_s": 60.743133, + "worker_capacity_series_present": true, + "jobs_ready_rate_1m": 0.0, + "diagnosis": "возраст старейшей задачи растёт при нулевой скорости обработки jobs (worker не отдаёт метрики) -> остановлен worker, а не медленная БД", + "alerts_in_state_firing": [ + "HighHttpErrorRate", + "HighUploadErrorRate", + "QueueLagGrowing" + ] + }, + "time_to_resolve_after_recovery_s": 21.8 + }, + { + "incident": "slow_db", + "alert": "HighLatencyP95", + "injected_at_s": 124.3, + "firing_at_s": 192.0, + "detection_seconds": 67.7, + "recovered_cmd_at_s": 193.7, + "resolved_at_s": 237.2, + "fired": true, + "resolved": true, + "evidence": { + "p95_s": 0.7828124999999998, + "db_pool_waiting": 0.0, + "slow_traces_with_long_db_span": 20, + "slow_http_post_messages_traces": 20, + "diagnosis": "p95 высокий, а время в спанах db.connection сопоставимо с длительностью запроса и растёт ожидание пула -> медленная БД, а не обработчик", + "alerts_in_state_firing": [ + "HighLatencyP95" + ] + }, + "time_to_resolve_after_recovery_s": 43.5 + }, + { + "incident": "s3_down", + "alert": "HighUploadErrorRate", + "injected_at_s": 243.2, + "firing_at_s": 327.3, + "detection_seconds": 84.1, + "recovered_cmd_at_s": 328.6, + "resolved_at_s": 367.9, + "fired": true, + "resolved": true, + "evidence": { + "upload_error_ratio_1m": 1.0, + "global_error_ratio_1m": 0.011166945840312675, + "error_traces": 20, + "s3_put_failed_log_lines": 37, + "diagnosis": "рост доли 5xx на маршруте загрузки при нормальных остальных; спаны s3.put_object с ошибкой и логи s3_put_failed -> недоступен S3", + "alerts_in_state_firing": [ + "HighUploadErrorRate", + "QueueLagGrowing" + ] + }, + "time_to_resolve_after_recovery_s": 39.3 + }, + { + "incident": "telemetry_backend", + "alert": "TelemetryExportFailing", + "injected_at_s": 373.9, + "firing_at_s": 442.9, + "detection_seconds": 69.1, + "recovered_cmd_at_s": 444.7, + "resolved_at_s": 512.2, + "fired": true, + "resolved": true, + "evidence": { + "api_error_ratio_1m": 0.0, + "api_rps": 62.62045892577912, + "collector_queue_size": 32.0, + "collector_send_failed_spans_rate": 62.94195947803441, + "collector_rss_mib": 239.0, + "diagnosis": "ошибки экспорта растут, очередь ограничена; бизнес-метрики API без роста ошибок -> сбой хранилища телеметрии, не приложения", + "alerts_in_state_firing": [ + "TelemetryExportFailing" + ] + }, + "time_to_resolve_after_recovery_s": 67.5 + } + ], + "traffic": { + "msg_2": 15471, + "up_2": 10598, + "up_5": 2175, + "up_err": 45 + } +} +exit=0 + +$ python -u research/obs_cost.py --out evidence\recheck_2026-11-12 --profiles off on100 head10 tail --repeat 2 +{"profile": "off", "post": {"requests": 41303, "ok": 40440, "errors": {"500": 863}, "seconds": 30, "ok_rps": 1348.0, "error_ratio": 0.0209, "p50_ms": 17.39, "p95_ms": 74.05, "p99_ms": 618.63}, "docker_stats": {"worker": {"avg_cpu_percent": 1.6, "peak_mem_mib": 54.6}, "api": {"avg_cpu_percent": 50.6, "peak_mem_mib": 76.4}, "otel-collector": {"avg_cpu_percent": 0.1, "peak_mem_mib": 45.1}, "tempo": {"avg_cpu_percent": 0.3, "peak_mem_mib": 426.2}, "lb": {"avg_cpu_percent": 23.0, "peak_mem_mib": 18.3}, "grafana": {"avg_cpu_percent": 1.1, "peak_mem_mib": 314.4}, "rabbitmq": {"avg_cpu_percent": 14.5, "peak_mem_mib": 177.9}, "postgres": {"avg_cpu_percent": 95.9, "peak_mem_mib": 264.3}, "loki": {"avg_cpu_percent": 0.5, "peak_mem_mib": 119.8}, "seaweedfs": {"avg_cpu_percent": 1.3, "peak_mem_mib": 524.7}, "prometheus": {"avg_cpu_percent": 0.7, "peak_mem_mib": 75.6}, "valkey": {"avg_cpu_percent": 4.7, "peak_mem_mib": 10.7}}, "collector_delta": {"receiver_accepted_spans": 0, "receiver_accepted_log_records": 0, "receiver_accepted_metric_points": 0, "receiver_refused_spans": 0, "exporter_sent_spans": 0, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 0, "loki_bytes_delta": 0, "errors_injected_500": 873, "slow_requests_gt500ms": 823, "traces_in_tempo": {"all_http": 0, "error": 0, "slow_gt500ms": 0}, "kept_ratio": {"errors": 0.0, "slow": 0.0}} +{"profile": "off", "post": {"requests": 42499, "ok": 41641, "errors": {"500": 858}, "seconds": 30, "ok_rps": 1388.0, "error_ratio": 0.0202, "p50_ms": 16.03, "p95_ms": 72.03, "p99_ms": 617.51}, "docker_stats": {"worker": {"avg_cpu_percent": 1.4, "peak_mem_mib": 54.6}, "api": {"avg_cpu_percent": 49.7, "peak_mem_mib": 64.0}, "otel-collector": {"avg_cpu_percent": 0.1, "peak_mem_mib": 43.7}, "tempo": {"avg_cpu_percent": 0.3, "peak_mem_mib": 388.5}, "lb": {"avg_cpu_percent": 22.6, "peak_mem_mib": 21.8}, "grafana": {"avg_cpu_percent": 1.4, "peak_mem_mib": 315.2}, "rabbitmq": {"avg_cpu_percent": 8.3, "peak_mem_mib": 178.2}, "postgres": {"avg_cpu_percent": 94.0, "peak_mem_mib": 263.0}, "loki": {"avg_cpu_percent": 0.5, "peak_mem_mib": 121.8}, "seaweedfs": {"avg_cpu_percent": 1.4, "peak_mem_mib": 505.8}, "prometheus": {"avg_cpu_percent": 0.6, "peak_mem_mib": 60.9}, "valkey": {"avg_cpu_percent": 4.5, "peak_mem_mib": 10.7}}, "collector_delta": {"receiver_accepted_spans": 0, "receiver_accepted_log_records": 0, "receiver_accepted_metric_points": 0, "receiver_refused_spans": 0, "exporter_sent_spans": 0, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 0, "loki_bytes_delta": 0, "errors_injected_500": 873, "slow_requests_gt500ms": 888, "traces_in_tempo": {"all_http": 0, "error": 0, "slow_gt500ms": 0}, "kept_ratio": {"errors": 0.0, "slow": 0.0}} +{"profile": "on100", "post": {"requests": 38267, "ok": 37530, "errors": {"500": 737}, "seconds": 30, "ok_rps": 1251.0, "error_ratio": 0.0193, "p50_ms": 24.79, "p95_ms": 97.91, "p99_ms": 631.54}, "docker_stats": {"worker": {"avg_cpu_percent": 2.2, "peak_mem_mib": 64.5}, "api": {"avg_cpu_percent": 80.6, "peak_mem_mib": 83.2}, "otel-collector": {"avg_cpu_percent": 3.4, "peak_mem_mib": 75.0}, "tempo": {"avg_cpu_percent": 11.2, "peak_mem_mib": 745.7}, "lb": {"avg_cpu_percent": 20.2, "peak_mem_mib": 21.5}, "grafana": {"avg_cpu_percent": 1.1, "peak_mem_mib": 320.0}, "rabbitmq": {"avg_cpu_percent": 12.1, "peak_mem_mib": 157.0}, "postgres": {"avg_cpu_percent": 92.6, "peak_mem_mib": 282.2}, "loki": {"avg_cpu_percent": 4.4, "peak_mem_mib": 194.8}, "seaweedfs": {"avg_cpu_percent": 1.4, "peak_mem_mib": 494.7}, "prometheus": {"avg_cpu_percent": 0.5, "peak_mem_mib": 60.8}, "valkey": {"avg_cpu_percent": 5.4, "peak_mem_mib": 11.5}}, "collector_delta": {"receiver_accepted_spans": 90462, "receiver_accepted_log_records": 44825, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 90462, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 47091712, "loki_bytes_delta": 30777344, "errors_injected_500": 761, "slow_requests_gt500ms": 808, "traces_in_tempo": {"all_http": 5000, "error": 848, "slow_gt500ms": 902}, "kept_ratio": {"errors": 1.114, "slow": 1.116}} +{"profile": "on100", "post": {"requests": 37970, "ok": 37178, "errors": {"500": 792}, "seconds": 30, "ok_rps": 1239.3, "error_ratio": 0.0209, "p50_ms": 24.47, "p95_ms": 102.73, "p99_ms": 627.88}, "docker_stats": {"worker": {"avg_cpu_percent": 1.7, "peak_mem_mib": 60.3}, "api": {"avg_cpu_percent": 78.4, "peak_mem_mib": 75.4}, "otel-collector": {"avg_cpu_percent": 3.3, "peak_mem_mib": 72.0}, "tempo": {"avg_cpu_percent": 7.9, "peak_mem_mib": 526.0}, "lb": {"avg_cpu_percent": 19.5, "peak_mem_mib": 21.7}, "grafana": {"avg_cpu_percent": 0.8, "peak_mem_mib": 321.6}, "rabbitmq": {"avg_cpu_percent": 12.0, "peak_mem_mib": 188.4}, "postgres": {"avg_cpu_percent": 89.5, "peak_mem_mib": 284.4}, "loki": {"avg_cpu_percent": 4.8, "peak_mem_mib": 216.5}, "seaweedfs": {"avg_cpu_percent": 1.3, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.7, "peak_mem_mib": 61.0}, "valkey": {"avg_cpu_percent": 5.0, "peak_mem_mib": 11.5}}, "collector_delta": {"receiver_accepted_spans": 89310, "receiver_accepted_log_records": 43824, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 89310, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 22323200, "loki_bytes_delta": 30490624, "errors_injected_500": 800, "slow_requests_gt500ms": 737, "traces_in_tempo": {"all_http": 5000, "error": 881, "slow_gt500ms": 850}, "kept_ratio": {"errors": 1.101, "slow": 1.153}} +{"profile": "head10", "post": {"requests": 34821, "ok": 34102, "errors": {"500": 719}, "seconds": 30, "ok_rps": 1136.7, "error_ratio": 0.0206, "p50_ms": 26.35, "p95_ms": 123.14, "p99_ms": 630.88}, "docker_stats": {"worker": {"avg_cpu_percent": 2.0, "peak_mem_mib": 62.5}, "api": {"avg_cpu_percent": 67.6, "peak_mem_mib": 83.7}, "otel-collector": {"avg_cpu_percent": 1.9, "peak_mem_mib": 68.4}, "tempo": {"avg_cpu_percent": 0.6, "peak_mem_mib": 472.9}, "lb": {"avg_cpu_percent": 19.2, "peak_mem_mib": 23.2}, "grafana": {"avg_cpu_percent": 0.9, "peak_mem_mib": 322.9}, "rabbitmq": {"avg_cpu_percent": 10.5, "peak_mem_mib": 194.3}, "postgres": {"avg_cpu_percent": 89.7, "peak_mem_mib": 302.1}, "loki": {"avg_cpu_percent": 4.8, "peak_mem_mib": 239.9}, "seaweedfs": {"avg_cpu_percent": 1.5, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.6, "peak_mem_mib": 60.9}, "valkey": {"avg_cpu_percent": 5.1, "peak_mem_mib": 12.8}}, "collector_delta": {"receiver_accepted_spans": 8506, "receiver_accepted_log_records": 40586, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 8506, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 1200128, "loki_bytes_delta": 4431872, "errors_injected_500": 727, "slow_requests_gt500ms": 671, "traces_in_tempo": {"all_http": 3919, "error": 89, "slow_gt500ms": 80}, "kept_ratio": {"errors": 0.122, "slow": 0.119}} +{"profile": "head10", "post": {"requests": 39840, "ok": 39078, "errors": {"500": 762}, "seconds": 30, "ok_rps": 1302.6, "error_ratio": 0.0191, "p50_ms": 21.48, "p95_ms": 79.8, "p99_ms": 622.79}, "docker_stats": {"worker": {"avg_cpu_percent": 1.3, "peak_mem_mib": 60.2}, "api": {"avg_cpu_percent": 69.4, "peak_mem_mib": 74.0}, "otel-collector": {"avg_cpu_percent": 1.9, "peak_mem_mib": 68.9}, "tempo": {"avg_cpu_percent": 0.6, "peak_mem_mib": 755.1}, "lb": {"avg_cpu_percent": 19.9, "peak_mem_mib": 21.7}, "grafana": {"avg_cpu_percent": 1.5, "peak_mem_mib": 325.1}, "rabbitmq": {"avg_cpu_percent": 10.9, "peak_mem_mib": 194.5}, "postgres": {"avg_cpu_percent": 91.7, "peak_mem_mib": 303.6}, "loki": {"avg_cpu_percent": 4.6, "peak_mem_mib": 281.0}, "seaweedfs": {"avg_cpu_percent": 1.2, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.7, "peak_mem_mib": 61.0}, "valkey": {"avg_cpu_percent": 5.0, "peak_mem_mib": 11.4}}, "collector_delta": {"receiver_accepted_spans": 9291, "receiver_accepted_log_records": 45766, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 9458, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 32768, "loki_bytes_delta": 32833536, "errors_injected_500": 774, "slow_requests_gt500ms": 802, "traces_in_tempo": {"all_http": 4411, "error": 81, "slow_gt500ms": 93}, "kept_ratio": {"errors": 0.105, "slow": 0.116}} +{"profile": "tail", "post": {"requests": 39275, "ok": 38469, "errors": {"500": 806}, "seconds": 30, "ok_rps": 1282.3, "error_ratio": 0.0205, "p50_ms": 23.74, "p95_ms": 86.59, "p99_ms": 631.13}, "docker_stats": {"worker": {"avg_cpu_percent": 2.1, "peak_mem_mib": 59.8}, "api": {"avg_cpu_percent": 77.1, "peak_mem_mib": 75.4}, "otel-collector": {"avg_cpu_percent": 4.6, "peak_mem_mib": 105.8}, "tempo": {"avg_cpu_percent": 0.8, "peak_mem_mib": 719.9}, "lb": {"avg_cpu_percent": 19.7, "peak_mem_mib": 22.0}, "grafana": {"avg_cpu_percent": 1.0, "peak_mem_mib": 325.2}, "rabbitmq": {"avg_cpu_percent": 12.5, "peak_mem_mib": 185.8}, "postgres": {"avg_cpu_percent": 93.0, "peak_mem_mib": 302.7}, "loki": {"avg_cpu_percent": 5.8, "peak_mem_mib": 312.2}, "seaweedfs": {"avg_cpu_percent": 1.4, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.6, "peak_mem_mib": 60.9}, "valkey": {"avg_cpu_percent": 5.1, "peak_mem_mib": 11.4}}, "collector_delta": {"receiver_accepted_spans": 91490, "receiver_accepted_log_records": 45024, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 13023, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 401408, "loki_bytes_delta": 34607104, "errors_injected_500": 818, "slow_requests_gt500ms": 793, "traces_in_tempo": {"all_http": 5000, "error": 907, "slow_gt500ms": 883}, "kept_ratio": {"errors": 1.109, "slow": 1.113}} +{"profile": "tail", "post": {"requests": 39821, "ok": 38989, "errors": {"500": 832}, "seconds": 30, "ok_rps": 1299.6, "error_ratio": 0.0209, "p50_ms": 23.35, "p95_ms": 93.86, "p99_ms": 626.16}, "docker_stats": {"worker": {"avg_cpu_percent": 1.3, "peak_mem_mib": 64.2}, "api": {"avg_cpu_percent": 77.7, "peak_mem_mib": 75.5}, "otel-collector": {"avg_cpu_percent": 3.4, "peak_mem_mib": 108.4}, "tempo": {"avg_cpu_percent": 1.9, "peak_mem_mib": 443.0}, "lb": {"avg_cpu_percent": 18.6, "peak_mem_mib": 21.6}, "grafana": {"avg_cpu_percent": 0.7, "peak_mem_mib": 325.1}, "rabbitmq": {"avg_cpu_percent": 9.5, "peak_mem_mib": 175.5}, "postgres": {"avg_cpu_percent": 88.4, "peak_mem_mib": 303.5}, "loki": {"avg_cpu_percent": 4.3, "peak_mem_mib": 317.3}, "seaweedfs": {"avg_cpu_percent": 4.2, "peak_mem_mib": 692.3}, "prometheus": {"avg_cpu_percent": 0.5, "peak_mem_mib": 61.1}, "valkey": {"avg_cpu_percent": 4.9, "peak_mem_mib": 11.7}}, "collector_delta": {"receiver_accepted_spans": 92273, "receiver_accepted_log_records": 45427, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 13252, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 1703936, "loki_bytes_delta": 32804864, "errors_injected_500": 837, "slow_requests_gt500ms": 743, "traces_in_tempo": {"all_http": 5000, "error": 925, "slow_gt500ms": 834}, "kept_ratio": {"errors": 1.105, "slow": 1.122}} +saved +exit=0 + +$ python research/plot_obs_cost.py --out evidence\recheck_2026-11-12 +saved +exit=0 + +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +exit=0 + +$ python -m unittest discover -s student-tests -p test_observability.py -v +test_logs_are_structured_and_linked_by_trace_id (test_observability.ObservabilityTest.test_logs_are_structured_and_linked_by_trace_id) ... ok +test_metrics_for_http_jobs_queue_pool_worker (test_observability.ObservabilityTest.test_metrics_for_http_jobs_queue_pool_worker) ... ok +test_no_high_cardinality_or_secret_labels (test_observability.ObservabilityTest.test_no_high_cardinality_or_secret_labels) ... ok +test_one_trace_spans_api_outbox_broker_and_worker (test_observability.ObservabilityTest.test_one_trace_spans_api_outbox_broker_and_worker) ... ok +test_secrets_and_message_bodies_are_not_exported (test_observability.ObservabilityTest.test_secrets_and_message_bodies_are_not_exported) ... ok + +---------------------------------------------------------------------- +Ran 5 tests in 12.547s + +OK +exit=0 diff --git a/evidence/recheck_2026-11-12/dataset.json b/evidence/recheck_2026-11-12/dataset.json new file mode 100644 index 0000000..777afbc --- /dev/null +++ b/evidence/recheck_2026-11-12/dataset.json @@ -0,0 +1,9 @@ +{ + "when": "after experiments and final smoke, 2026-11-12", + "rows": { + "users": 739, + "chats": 425, + "messages": 1610549, + "attachments": 68898 + } +} \ No newline at end of file diff --git a/evidence/recheck_2026-11-12/failover.json b/evidence/recheck_2026-11-12/failover.json new file mode 100644 index 0000000..8283a4f --- /dev/null +++ b/evidence/recheck_2026-11-12/failover.json @@ -0,0 +1,10 @@ +{ + "stopped_in_order": [ + "mtusi-lab6-api-1", + "mtusi-lab6-api-2" + ], + "distinct_replicas": [ + "mtusi-lab6-api-1", + "mtusi-lab6-api-2" + ] +} \ No newline at end of file diff --git a/evidence/recheck_2026-11-12/grafana_provisioning.json b/evidence/recheck_2026-11-12/grafana_provisioning.json new file mode 100644 index 0000000..9168fd8 --- /dev/null +++ b/evidence/recheck_2026-11-12/grafana_provisioning.json @@ -0,0 +1,25 @@ +{ + "datasources": [ + { + "name": "Loki", + "type": "loki", + "url": "http://loki:3100" + }, + { + "name": "Prometheus", + "type": "prometheus", + "url": "http://prometheus:9090" + }, + { + "name": "Tempo", + "type": "tempo", + "url": "http://tempo:3200" + } + ], + "dashboard": { + "uid": "messenger-overview", + "title": "Messenger: API, очередь, worker", + "provisioned": true, + "panels": 22 + } +} \ No newline at end of file diff --git a/evidence/recheck_2026-11-12/incidents.json b/evidence/recheck_2026-11-12/incidents.json new file mode 100644 index 0000000..b41a7ba --- /dev/null +++ b/evidence/recheck_2026-11-12/incidents.json @@ -0,0 +1,1563 @@ +{ + "date": "2026-11-12 04:36:46", + "traffic_counters": { + "msg_2": 15471, + "up_2": 10598, + "up_5": 2175, + "up_err": 45 + }, + "incidents": [ + { + "incident": "worker_down", + "alert": "QueueLagGrowing", + "injected_at_s": 25.0, + "firing_at_s": 93.3, + "detection_seconds": 68.3, + "recovered_cmd_at_s": 96.5, + "resolved_at_s": 118.3, + "fired": true, + "resolved": true, + "evidence": { + "queue_oldest_age_s": 60.743133, + "worker_capacity_series_present": true, + "jobs_ready_rate_1m": 0.0, + "diagnosis": "возраст старейшей задачи растёт при нулевой скорости обработки jobs (worker не отдаёт метрики) -> остановлен worker, а не медленная БД", + "alerts_in_state_firing": [ + "HighHttpErrorRate", + "HighUploadErrorRate", + "QueueLagGrowing" + ] + }, + "time_to_resolve_after_recovery_s": 21.8 + }, + { + "incident": "slow_db", + "alert": "HighLatencyP95", + "injected_at_s": 124.3, + "firing_at_s": 192.0, + "detection_seconds": 67.7, + "recovered_cmd_at_s": 193.7, + "resolved_at_s": 237.2, + "fired": true, + "resolved": true, + "evidence": { + "p95_s": 0.7828124999999998, + "db_pool_waiting": 0.0, + "slow_traces_with_long_db_span": 20, + "slow_http_post_messages_traces": 20, + "diagnosis": "p95 высокий, а время в спанах db.connection сопоставимо с длительностью запроса и растёт ожидание пула -> медленная БД, а не обработчик", + "alerts_in_state_firing": [ + "HighLatencyP95" + ] + }, + "time_to_resolve_after_recovery_s": 43.5 + }, + { + "incident": "s3_down", + "alert": "HighUploadErrorRate", + "injected_at_s": 243.2, + "firing_at_s": 327.3, + "detection_seconds": 84.1, + "recovered_cmd_at_s": 328.6, + "resolved_at_s": 367.9, + "fired": true, + "resolved": true, + "evidence": { + "upload_error_ratio_1m": 1.0, + "global_error_ratio_1m": 0.011166945840312675, + "error_traces": 20, + "s3_put_failed_log_lines": 37, + "diagnosis": "рост доли 5xx на маршруте загрузки при нормальных остальных; спаны s3.put_object с ошибкой и логи s3_put_failed -> недоступен S3", + "alerts_in_state_firing": [ + "HighUploadErrorRate", + "QueueLagGrowing" + ] + }, + "time_to_resolve_after_recovery_s": 39.3 + }, + { + "incident": "telemetry_backend", + "alert": "TelemetryExportFailing", + "injected_at_s": 373.9, + "firing_at_s": 442.9, + "detection_seconds": 69.1, + "recovered_cmd_at_s": 444.7, + "resolved_at_s": 512.2, + "fired": true, + "resolved": true, + "evidence": { + "api_error_ratio_1m": 0.0, + "api_rps": 62.62045892577912, + "collector_queue_size": 32.0, + "collector_send_failed_spans_rate": 62.94195947803441, + "collector_rss_mib": 239.0, + "diagnosis": "ошибки экспорта растут, очередь ограничена; бизнес-метрики API без роста ошибок -> сбой хранилища телеметрии, не приложения", + "alerts_in_state_firing": [ + "TelemetryExportFailing" + ] + }, + "time_to_resolve_after_recovery_s": 67.5 + } + ], + "timeline": [ + { + "t": 0.0, + "alerts": {}, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.0, + "p95_s": NaN, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 0.0 + }, + { + "t": 4.2, + "alerts": {}, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.0, + "p95_s": NaN, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 0.0 + }, + { + "t": 8.4, + "alerts": {}, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.0, + "p95_s": NaN, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 0.0 + }, + { + "t": 12.6, + "alerts": {}, + "queue_oldest_age_s": 0.034891, + "error_ratio_1m": 0.0, + "p95_s": 0.15999999999999967, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 2.4000872759009413 + }, + { + "t": 16.7, + "alerts": {}, + "queue_oldest_age_s": 0.139, + "error_ratio_1m": 0.0, + "p95_s": 0.15999999999999967, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 2.4000872759009413 + }, + { + "t": 20.9, + "alerts": {}, + "queue_oldest_age_s": 0.051647, + "error_ratio_1m": 0.0, + "p95_s": 0.026236715590362808, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 8.336762548941174 + }, + { + "t": 25.0, + "alerts": {}, + "queue_oldest_age_s": 0.115789, + "error_ratio_1m": 0.0, + "p95_s": 0.02442354818368139, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 14.430351655544849 + }, + { + "t": 29.2, + "alerts": {}, + "queue_oldest_age_s": 0.545283, + "error_ratio_1m": 0.0, + "p95_s": 0.024115008903037197, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 20.541318169145544 + }, + { + "t": 33.3, + "alerts": {}, + "queue_oldest_age_s": 0.545283, + "error_ratio_1m": 0.0, + "p95_s": 0.02401965194518706, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 26.636230052667628 + }, + { + "t": 37.4, + "alerts": {}, + "queue_oldest_age_s": 5.675415, + "error_ratio_1m": 0.0, + "p95_s": 0.02424849113518554, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 32.607982714522954 + }, + { + "t": 41.6, + "alerts": {}, + "queue_oldest_age_s": 10.562799, + "error_ratio_1m": 0.0, + "p95_s": 0.02424849113518554, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 32.607982714522954 + }, + { + "t": 45.7, + "alerts": {}, + "queue_oldest_age_s": 15.71053, + "error_ratio_1m": 0.0, + "p95_s": 0.024084642500017767, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 38.77484453980145 + }, + { + "t": 49.8, + "alerts": {}, + "queue_oldest_age_s": 20.593022, + "error_ratio_1m": 0.0571804785866381, + "p95_s": 0.023807941318499344, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 44.723913886425365 + }, + { + "t": 53.9, + "alerts": { + "HighHttpErrorRate": "pending" + }, + "queue_oldest_age_s": 25.752607, + "error_ratio_1m": 0.1342881907407185, + "p95_s": 0.023456234665698433, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 52.43801128049771 + }, + { + "t": 58.1, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "pending" + }, + "queue_oldest_age_s": 25.752607, + "error_ratio_1m": 0.16614782866088135, + "p95_s": 0.023182652522159137, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 58.421724384907705 + }, + { + "t": 62.3, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "pending" + }, + "queue_oldest_age_s": 30.636563, + "error_ratio_1m": 0.16614782866088135, + "p95_s": 0.023182652522159137, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 64.41148201696144 + }, + { + "t": 66.4, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "pending" + }, + "queue_oldest_age_s": 35.781885, + "error_ratio_1m": 0.19328511458700884, + "p95_s": 0.022905959297194516, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 64.41148201696144 + }, + { + "t": 70.5, + "alerts": { + "HighHttpErrorRate": "pending", + "HighUploadErrorRate": "pending", + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 40.667245, + "error_ratio_1m": 0.2206347703908339, + "p95_s": 0.02225033148542708, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.45104506778685 + }, + { + "t": 74.7, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 45.817961, + "error_ratio_1m": 0.2601124575093317, + "p95_s": 0.021541387006281255, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.51537739068002 + }, + { + "t": 78.8, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 50.701294, + "error_ratio_1m": 0.30006224110612695, + "p95_s": 0.020729469406863467, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.37095862443115 + }, + { + "t": 82.9, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 50.701294, + "error_ratio_1m": 0.3402474698111322, + "p95_s": 0.019374236760736192, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.23867348629874 + }, + { + "t": 87.0, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 55.864793, + "error_ratio_1m": 0.3402474698111322, + "p95_s": 0.019374236760736192, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.15720554987078 + }, + { + "t": 91.1, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 60.743133, + "error_ratio_1m": 0.38108741722062406, + "p95_s": 0.016374973142925298, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.15720554987078 + }, + { + "t": 95.2, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 65.89879, + "error_ratio_1m": 0.4203257696841435, + "p95_s": 0.009901139146070291, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.15988289495357 + }, + { + "t": 99.4, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 70.779215, + "error_ratio_1m": 0.4616670906891614, + "p95_s": 0.009360996461506996, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 67.85750753410055 + }, + { + "t": 103.6, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 70.779215, + "error_ratio_1m": 0.47148891235480467, + "p95_s": 0.009311411992263055, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 68.87523182661188 + }, + { + "t": 107.8, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 70.046846, + "error_ratio_1m": 0.471394674400211, + "p95_s": 0.009333094555873923, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 68.96614422342631 + }, + { + "t": 112.0, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 2.200657, + "error_ratio_1m": 0.471394674400211, + "p95_s": 0.009333094555873923, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 68.85704934724899 + }, + { + "t": 116.1, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 0.004353, + "error_ratio_1m": 0.4317401637179826, + "p95_s": 0.009875472589792057, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 68.85704934724899 + }, + { + "t": 120.3, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.043888, + "error_ratio_1m": 0.38874207188160675, + "p95_s": 0.01581978798586575, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 68.80250190916034 + }, + { + "t": 124.5, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.029942, + "error_ratio_1m": 0.34718179412543004, + "p95_s": 0.0186427680798005, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 68.71034018800343 + }, + { + "t": 128.8, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.345868, + "error_ratio_1m": 0.3049363057324841, + "p95_s": 0.02019075144508671, + "db_waiting": 0.0, + "db_used": 7.0, + "rps": 68.51407375081826 + }, + { + "t": 133.0, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.345868, + "error_ratio_1m": 0.262221041445271, + "p95_s": 0.021144215530903322, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 68.44258568960815 + }, + { + "t": 137.3, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.655514, + "error_ratio_1m": 0.262221041445271, + "p95_s": 0.021144215530903322, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 66.75030911339007 + }, + { + "t": 141.4, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.839632, + "error_ratio_1m": 0.224734404794334, + "p95_s": 0.022209459459459457, + "db_waiting": 0.0, + "db_used": 8.0, + "rps": 66.75030911339007 + }, + { + "t": 145.6, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.734771, + "error_ratio_1m": 0.1890575766256087, + "p95_s": 0.023875, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 63.477343806822326 + }, + { + "t": 149.8, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.357819, + "error_ratio_1m": 0.15070975536091816, + "p95_s": 0.029673295454545452, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 60.203283815480845 + }, + { + "t": 154.1, + "alerts": { + "HighHttpErrorRate": "firing", + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.059194, + "error_ratio_1m": 0.10764331210191083, + "p95_s": 0.044417475728155284, + "db_waiting": 0.0, + "db_used": 6.0, + "rps": 57.0929851994618 + }, + { + "t": 158.3, + "alerts": { + "HighHttpErrorRate": "firing" + }, + "queue_oldest_age_s": 0.059194, + "error_ratio_1m": 0.058625336927223715, + "p95_s": 0.09520000000000031, + "db_waiting": 0.0, + "db_used": 8.0, + "rps": 53.96559874904541 + }, + { + "t": 162.4, + "alerts": { + "HighHttpErrorRate": "firing" + }, + "queue_oldest_age_s": 0.070462, + "error_ratio_1m": 0.058625336927223715, + "p95_s": 0.1788999999999996, + "db_waiting": 0.0, + "db_used": 8.0, + "rps": 50.80277106023965 + }, + { + "t": 166.6, + "alerts": { + "HighHttpErrorRate": "firing" + }, + "queue_oldest_age_s": 0.21636, + "error_ratio_1m": 0.003579098067287044, + "p95_s": 0.1788999999999996, + "db_waiting": 0.0, + "db_used": 6.0, + "rps": 50.80277106023965 + }, + { + "t": 170.7, + "alerts": {}, + "queue_oldest_age_s": 0.105469, + "error_ratio_1m": 0.0, + "p95_s": 0.5317857142857132, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 47.67446088948688 + }, + { + "t": 174.9, + "alerts": { + "HighLatencyP95": "pending" + }, + "queue_oldest_age_s": 0.281659, + "error_ratio_1m": 0.0, + "p95_s": 0.6167187499999996, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 44.60243285997418 + }, + { + "t": 179.0, + "alerts": { + "HighLatencyP95": "pending" + }, + "queue_oldest_age_s": 0.264954, + "error_ratio_1m": 0.0, + "p95_s": 0.6852209944751378, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 41.43862392494136 + }, + { + "t": 183.2, + "alerts": { + "HighLatencyP95": "pending" + }, + "queue_oldest_age_s": 0.264954, + "error_ratio_1m": 0.0, + "p95_s": 0.740563725490196, + "db_waiting": 0.0, + "db_used": 6.0, + "rps": 38.490209268922385 + }, + { + "t": 187.5, + "alerts": { + "HighLatencyP95": "pending" + }, + "queue_oldest_age_s": 0.525214, + "error_ratio_1m": 0.0, + "p95_s": 0.7828124999999998, + "db_waiting": 0.0, + "db_used": 6.0, + "rps": 35.38246149929999 + }, + { + "t": 191.9, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.452948, + "error_ratio_1m": 0.0, + "p95_s": 0.7828124999999998, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 33.872727272727275 + }, + { + "t": 196.1, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.52194, + "error_ratio_1m": 0.0, + "p95_s": 0.8059375, + "db_waiting": 0.0, + "db_used": 8.0, + "rps": 33.872727272727275 + }, + { + "t": 200.4, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.015883, + "error_ratio_1m": 0.0, + "p95_s": 0.802436440677966, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 33.90909090909091 + }, + { + "t": 204.6, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.172855, + "error_ratio_1m": 0.0, + "p95_s": 0.8053125000000002, + "db_waiting": 0.0, + "db_used": 6.0, + "rps": 33.98243604429172 + }, + { + "t": 208.8, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.283117, + "error_ratio_1m": 0.0, + "p95_s": 0.7842511013215863, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 35.61882943326243 + }, + { + "t": 212.9, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.283117, + "error_ratio_1m": 0.0, + "p95_s": 0.7439950980392154, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 37.983199389068695 + }, + { + "t": 217.1, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.0, + "p95_s": 0.7439950980392154, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 40.437834103058286 + }, + { + "t": 221.2, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.2148, + "error_ratio_1m": 0.0, + "p95_s": 0.697826086956521, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 40.437834103058286 + }, + { + "t": 225.4, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.211136, + "error_ratio_1m": 0.0, + "p95_s": 0.6292187499999996, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 43.147808062257944 + }, + { + "t": 229.6, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.353579, + "error_ratio_1m": 0.0, + "p95_s": 0.5499999999999998, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 45.81984799447252 + }, + { + "t": 233.7, + "alerts": { + "HighLatencyP95": "firing" + }, + "queue_oldest_age_s": 0.245786, + "error_ratio_1m": 0.0, + "p95_s": 0.23112288135593195, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 48.67449725444561 + }, + { + "t": 237.9, + "alerts": {}, + "queue_oldest_age_s": 0.245786, + "error_ratio_1m": 0.0, + "p95_s": 0.1951446280991737, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 51.002781969925636 + }, + { + "t": 242.1, + "alerts": {}, + "queue_oldest_age_s": 0.282524, + "error_ratio_1m": 0.0, + "p95_s": 0.1951446280991737, + "db_waiting": 0.0, + "db_used": 8.0, + "rps": 53.58279241440754 + }, + { + "t": 246.2, + "alerts": {}, + "queue_oldest_age_s": 0.026938, + "error_ratio_1m": 0.0, + "p95_s": 0.17567708333333332, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 53.58279241440754 + }, + { + "t": 250.4, + "alerts": {}, + "queue_oldest_age_s": 3.268197, + "error_ratio_1m": 0.0, + "p95_s": 0.14193661971830948, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 56.11215157190392 + }, + { + "t": 254.6, + "alerts": {}, + "queue_oldest_age_s": 8.299419, + "error_ratio_1m": 0.0, + "p95_s": 0.11426573426573376, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 58.69411058785025 + }, + { + "t": 258.7, + "alerts": {}, + "queue_oldest_age_s": 13.272433, + "error_ratio_1m": 0.001528584530724549, + "p95_s": 0.09632575757575763, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 59.47597141662273 + }, + { + "t": 263.0, + "alerts": {}, + "queue_oldest_age_s": 13.272433, + "error_ratio_1m": 0.0034580320653882423, + "p95_s": 0.09800847457627111, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 57.83846685334012 + }, + { + "t": 267.1, + "alerts": {}, + "queue_oldest_age_s": 18.303537, + "error_ratio_1m": 0.0034580320653882423, + "p95_s": 0.09800847457627111, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 55.40201461871341 + }, + { + "t": 271.3, + "alerts": {}, + "queue_oldest_age_s": 23.278544, + "error_ratio_1m": 0.0049228749589760425, + "p95_s": 0.09180167597765354, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 55.40201461871341 + }, + { + "t": 275.4, + "alerts": {}, + "queue_oldest_age_s": 28.308455, + "error_ratio_1m": 0.005165289256198348, + "p95_s": 0.08781065088757387, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 52.80192006982072 + }, + { + "t": 279.5, + "alerts": {}, + "queue_oldest_age_s": 33.282528, + "error_ratio_1m": 0.006919155134741442, + "p95_s": 0.08984756097560971, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 49.929088330484745 + }, + { + "t": 283.6, + "alerts": {}, + "queue_oldest_age_s": 38.31274, + "error_ratio_1m": 0.008104978772674644, + "p95_s": 0.09164965986394565, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 47.11080402923743 + }, + { + "t": 287.7, + "alerts": { + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 38.31274, + "error_ratio_1m": 0.008638420403126287, + "p95_s": 0.09352083333333336, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 44.20160733117568 + }, + { + "t": 291.9, + "alerts": { + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 43.28704, + "error_ratio_1m": 0.008638420403126287, + "p95_s": 0.09352083333333336, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 44.20160733117568 + }, + { + "t": 296.0, + "alerts": { + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 48.317993, + "error_ratio_1m": 0.010034904013961607, + "p95_s": 0.08963302752293567, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 41.674242699734535 + }, + { + "t": 300.2, + "alerts": { + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 53.292534, + "error_ratio_1m": 0.011649580615097856, + "p95_s": 0.08432038834951461, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 39.019600712753196 + }, + { + "t": 304.4, + "alerts": { + "QueueLagGrowing": "pending" + }, + "queue_oldest_age_s": 58.321112, + "error_ratio_1m": 0.013506753376688345, + "p95_s": 0.0846474358974357, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 36.34611538391607 + }, + { + "t": 308.5, + "alerts": { + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 58.321112, + "error_ratio_1m": 0.01576086956521739, + "p95_s": 0.09680851063829765, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 33.455762027710094 + }, + { + "t": 312.7, + "alerts": { + "HighUploadErrorRate": "pending", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 63.296048, + "error_ratio_1m": 0.013392857142857142, + "p95_s": 0.09590909090909094, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 32.58300301829158 + }, + { + "t": 316.9, + "alerts": { + "HighUploadErrorRate": "pending", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 68.325599, + "error_ratio_1m": 0.013392857142857142, + "p95_s": 0.09590909090909094, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 32.565412658872305 + }, + { + "t": 321.1, + "alerts": { + "HighUploadErrorRate": "pending", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 73.302234, + "error_ratio_1m": 0.011166945840312675, + "p95_s": 0.08134615384615385, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 32.565412658872305 + }, + { + "t": 325.2, + "alerts": { + "HighUploadErrorRate": "pending", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 78.332693, + "error_ratio_1m": 0.011166945840312675, + "p95_s": 0.08306249999999998, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 32.565412658872305 + }, + { + "t": 329.3, + "alerts": { + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 83.305897, + "error_ratio_1m": 0.011160714285714286, + "p95_s": 0.08219512195121911, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 32.58300301829157 + }, + { + "t": 333.5, + "alerts": { + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 83.305897, + "error_ratio_1m": 0.010011123470522802, + "p95_s": 0.07154761904761914, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 32.692097894468894 + }, + { + "t": 337.7, + "alerts": { + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 87.306779, + "error_ratio_1m": 0.008903728436282694, + "p95_s": 0.065654761904762, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 32.67450951870102 + }, + { + "t": 341.8, + "alerts": { + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.008903728436282694, + "p95_s": 0.065654761904762, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 32.78360637852974 + }, + { + "t": 346.0, + "alerts": { + "HighUploadErrorRate": "firing", + "QueueLagGrowing": "firing" + }, + "queue_oldest_age_s": 0.0, + "error_ratio_1m": 0.012201885745978925, + "p95_s": 0.07484999999999997, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 32.78360637852974 + }, + { + "t": 350.2, + "alerts": { + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.515701, + "error_ratio_1m": 0.02352297592997812, + "p95_s": 0.09437499999999992, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 33.23757227535547 + }, + { + "t": 354.5, + "alerts": { + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.345487, + "error_ratio_1m": 0.035997882477501325, + "p95_s": 0.16877659574467965, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 34.34670351649151 + }, + { + "t": 358.7, + "alerts": { + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.186808, + "error_ratio_1m": 0.0323846908734053, + "p95_s": 0.1386170212765963, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 37.05589294156152 + }, + { + "t": 362.9, + "alerts": { + "HighUploadErrorRate": "firing" + }, + "queue_oldest_age_s": 0.186808, + "error_ratio_1m": 0.02907769195820082, + "p95_s": 0.11484999999999881, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 40.018909434716996 + }, + { + "t": 367.0, + "alerts": {}, + "queue_oldest_age_s": 0.34578, + "error_ratio_1m": 0.02907769195820082, + "p95_s": 0.11484999999999881, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 42.856103858322115 + }, + { + "t": 371.2, + "alerts": {}, + "queue_oldest_age_s": 0.29709, + "error_ratio_1m": 0.027153160797624098, + "p95_s": 0.10042452830188706, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 42.856103858322115 + }, + { + "t": 375.4, + "alerts": {}, + "queue_oldest_age_s": 0.136093, + "error_ratio_1m": 0.024800000000000003, + "p95_s": 0.10238095238095224, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 45.4561984072148 + }, + { + "t": 379.6, + "alerts": {}, + "queue_oldest_age_s": 0.392272, + "error_ratio_1m": 0.02189505473763684, + "p95_s": 0.09569548872180451, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 48.16538783228481 + }, + { + "t": 383.8, + "alerts": {}, + "queue_oldest_age_s": 0.158114, + "error_ratio_1m": 0.020743919885550785, + "p95_s": 0.0993706293706294, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 50.83821229862904 + }, + { + "t": 388.0, + "alerts": {}, + "queue_oldest_age_s": 0.158114, + "error_ratio_1m": 0.019034670292318152, + "p95_s": 0.11067010309278322, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 53.49285428561038 + }, + { + "t": 392.1, + "alerts": {}, + "queue_oldest_age_s": 0.125731, + "error_ratio_1m": 0.019034670292318152, + "p95_s": 0.11067010309278322, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 56.42023346303502 + }, + { + "t": 396.3, + "alerts": {}, + "queue_oldest_age_s": 0.361732, + "error_ratio_1m": 0.018047051240734774, + "p95_s": 0.10706310679611597, + "db_waiting": 0.0, + "db_used": 6.0, + "rps": 56.42023346303502 + }, + { + "t": 400.5, + "alerts": {}, + "queue_oldest_age_s": 0.242907, + "error_ratio_1m": 0.01537042729787888, + "p95_s": 0.09866379310344839, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 59.148680837136574 + }, + { + "t": 404.7, + "alerts": {}, + "queue_oldest_age_s": 0.476846, + "error_ratio_1m": 0.008033323415650105, + "p95_s": 0.09657670454545447, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 61.11242431405349 + }, + { + "t": 408.8, + "alerts": {}, + "queue_oldest_age_s": 0.077676, + "error_ratio_1m": 0.0, + "p95_s": 0.08937134502923962, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 62.676145971598444 + }, + { + "t": 413.0, + "alerts": {}, + "queue_oldest_age_s": 0.077676, + "error_ratio_1m": 0.0, + "p95_s": 0.09109374999999983, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 62.73069440151281 + }, + { + "t": 417.2, + "alerts": {}, + "queue_oldest_age_s": 0.138413, + "error_ratio_1m": 0.0, + "p95_s": 0.09109374999999983, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 62.62159754168409 + }, + { + "t": 421.4, + "alerts": {}, + "queue_oldest_age_s": 0.300125, + "error_ratio_1m": 0.0, + "p95_s": 0.09049999999999993, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 62.62159754168409 + }, + { + "t": 425.7, + "alerts": {}, + "queue_oldest_age_s": 0.211413, + "error_ratio_1m": 0.0, + "p95_s": 0.08950549450549455, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 62.61932035127912 + }, + { + "t": 430.0, + "alerts": { + "TelemetryExportFailing": "pending" + }, + "queue_oldest_age_s": 0.489429, + "error_ratio_1m": 0.0, + "p95_s": 0.08936746987951785, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 62.65682388450489 + }, + { + "t": 434.2, + "alerts": { + "TelemetryExportFailing": "pending" + }, + "queue_oldest_age_s": 0.110888, + "error_ratio_1m": 0.0, + "p95_s": 0.09392424242424234, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 62.567049111769734 + }, + { + "t": 438.3, + "alerts": { + "TelemetryExportFailing": "pending" + }, + "queue_oldest_age_s": 0.110888, + "error_ratio_1m": 0.0, + "p95_s": 0.0891558441558441, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 62.6943287815699 + }, + { + "t": 442.5, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.261078, + "error_ratio_1m": 0.0, + "p95_s": 0.08765714285714271, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 62.62045892577912 + }, + { + "t": 446.6, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.228472, + "error_ratio_1m": 0.0, + "p95_s": 0.08765714285714271, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 62.62045892577912 + }, + { + "t": 450.8, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.473824, + "error_ratio_1m": 0.0, + "p95_s": 0.09226608187134495, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 62.34772173533582 + }, + { + "t": 455.0, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.224432, + "error_ratio_1m": 0.0, + "p95_s": 0.09413043478260869, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 62.14545454545454 + }, + { + "t": 459.2, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.390557, + "error_ratio_1m": 0.0, + "p95_s": 0.09487499999999988, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 62.33067258214084 + }, + { + "t": 463.4, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.390557, + "error_ratio_1m": 0.0, + "p95_s": 0.09697852760736202, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 62.13066167245486 + }, + { + "t": 467.6, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.130839, + "error_ratio_1m": 0.0, + "p95_s": 0.09658045977011478, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 62.14884448242631 + }, + { + "t": 471.8, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.241717, + "error_ratio_1m": 0.0, + "p95_s": 0.09658045977011478, + "db_waiting": 0.0, + "db_used": 2.0, + "rps": 62.32953925597293 + }, + { + "t": 476.0, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.057754, + "error_ratio_1m": 0.0, + "p95_s": 0.09345029239766074, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 62.32953925597293 + }, + { + "t": 480.2, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.19608, + "error_ratio_1m": 0.0, + "p95_s": 0.09799707602339174, + "db_waiting": 0.0, + "db_used": 11.0, + "rps": 62.12953198298119 + }, + { + "t": 484.6, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.265437, + "error_ratio_1m": 0.0, + "p95_s": 0.09977941176470581, + "db_waiting": 0.0, + "db_used": 5.0, + "rps": 62.092038037055225 + }, + { + "t": 488.8, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.253323, + "error_ratio_1m": 0.0, + "p95_s": 0.10234104046242778, + "db_waiting": 0.0, + "db_used": 1.0, + "rps": 61.92952470998946 + }, + { + "t": 493.0, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.253323, + "error_ratio_1m": 0.0, + "p95_s": 0.12063451776649758, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 61.78406487508637 + }, + { + "t": 497.2, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.354343, + "error_ratio_1m": 0.0, + "p95_s": 0.12063451776649758, + "db_waiting": 0.0, + "db_used": 3.0, + "rps": 61.82155390294016 + }, + { + "t": 501.4, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.212754, + "error_ratio_1m": 0.0, + "p95_s": 0.130841121495327, + "db_waiting": 0.0, + "db_used": 4.0, + "rps": 61.82155390294016 + }, + { + "t": 505.6, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.345723, + "error_ratio_1m": 0.0, + "p95_s": 0.1253308823529411, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 61.656787519546164 + }, + { + "t": 509.7, + "alerts": { + "TelemetryExportFailing": "firing" + }, + "queue_oldest_age_s": 0.327575, + "error_ratio_1m": 0.0, + "p95_s": 0.12335820895522387, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 61.71245704311144 + }, + { + "t": 514.0, + "alerts": {}, + "queue_oldest_age_s": 0.358395, + "error_ratio_1m": 0.0, + "p95_s": 0.11286290322580625, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 61.83748795432645 + }, + { + "t": 518.1, + "alerts": {}, + "queue_oldest_age_s": 0.358395, + "error_ratio_1m": 0.0, + "p95_s": 0.10538135593220341, + "db_waiting": 0.0, + "db_used": 0.0, + "rps": 62.05567373952255 + } + ] +} \ No newline at end of file diff --git a/evidence/recheck_2026-11-12/incidents.png b/evidence/recheck_2026-11-12/incidents.png new file mode 100644 index 0000000..f329190 Binary files /dev/null and b/evidence/recheck_2026-11-12/incidents.png differ diff --git a/evidence/recheck_2026-11-12/lifecycle.json b/evidence/recheck_2026-11-12/lifecycle.json new file mode 100644 index 0000000..a975653 --- /dev/null +++ b/evidence/recheck_2026-11-12/lifecycle.json @@ -0,0 +1,274 @@ +{ + "before": { + "mtusi-lab6-api-1": { + "service": "api", + "started_at": "2026-11-12T01:25:27.883010338Z", + "running": true, + "health": "healthy", + "volumes": [], + "memory_bytes": 268435456, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-api-2": { + "service": "api", + "started_at": "2026-11-12T01:25:28.11093455Z", + "running": true, + "health": "healthy", + "volumes": [], + "memory_bytes": 268435456, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-grafana-1": { + "service": "grafana", + "started_at": "2026-11-12T01:21:50.391698676Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_grafanadata" + ], + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-lb-1": { + "service": "lb", + "started_at": "2026-11-12T01:21:48.668160902Z", + "running": true, + "health": "healthy", + "volumes": [], + "memory_bytes": 134217728, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-loki-1": { + "service": "loki", + "started_at": "2026-11-12T01:21:50.190375514Z", + "running": true, + "health": null, + "volumes": [ + "mtusi-lab6_lokidata" + ], + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-otel-collector-1": { + "service": "otel-collector", + "started_at": "2026-11-12T01:21:51.528830632Z", + "running": true, + "health": null, + "volumes": [], + "memory_bytes": 402653184, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-postgres-1": { + "service": "postgres", + "started_at": "2026-11-12T01:21:50.009875096Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_pgdata" + ], + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-prometheus-1": { + "service": "prometheus", + "started_at": "2026-11-12T01:21:49.578549023Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_promdata" + ], + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-rabbitmq-1": { + "service": "rabbitmq", + "started_at": "2026-11-12T01:21:51.306616597Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_rabbitmqdata" + ], + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-seaweedfs-1": { + "service": "seaweedfs", + "started_at": "2026-11-12T01:21:50.815349111Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_seaweeddata" + ], + "memory_bytes": 805306368, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-tempo-1": { + "service": "tempo", + "started_at": "2026-11-12T01:21:51.015757118Z", + "running": true, + "health": null, + "volumes": [ + "mtusi-lab6_tempodata" + ], + "memory_bytes": 1073741824, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-valkey-1": { + "service": "valkey", + "started_at": "2026-11-12T01:21:49.795208905Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_valkeydata" + ], + "memory_bytes": 134217728, + "nano_cpus": 500000000 + }, + "mtusi-lab6-worker-1": { + "service": "worker", + "started_at": "2026-11-12T01:25:27.886972662Z", + "running": true, + "health": "healthy", + "volumes": [], + "memory_bytes": 805306368, + "nano_cpus": 1000000000 + } + }, + "after": { + "mtusi-lab6-api-1": { + "service": "api", + "started_at": "2026-11-12T01:25:43.517609672Z", + "running": true, + "health": "healthy", + "volumes": [], + "memory_bytes": 268435456, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-api-2": { + "service": "api", + "started_at": "2026-11-12T01:25:43.743154951Z", + "running": true, + "health": "healthy", + "volumes": [], + "memory_bytes": 268435456, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-grafana-1": { + "service": "grafana", + "started_at": "2026-11-12T01:25:37.85991836Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_grafanadata" + ], + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-lb-1": { + "service": "lb", + "started_at": "2026-11-12T01:25:36.297842763Z", + "running": true, + "health": "healthy", + "volumes": [], + "memory_bytes": 134217728, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-loki-1": { + "service": "loki", + "started_at": "2026-11-12T01:25:37.451077579Z", + "running": true, + "health": null, + "volumes": [ + "mtusi-lab6_lokidata" + ], + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-otel-collector-1": { + "service": "otel-collector", + "started_at": "2026-11-12T01:25:39.004879376Z", + "running": true, + "health": null, + "volumes": [], + "memory_bytes": 402653184, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-postgres-1": { + "service": "postgres", + "started_at": "2026-11-12T01:25:37.693692485Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_pgdata" + ], + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-prometheus-1": { + "service": "prometheus", + "started_at": "2026-11-12T01:25:37.087818096Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_promdata" + ], + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-rabbitmq-1": { + "service": "rabbitmq", + "started_at": "2026-11-12T01:25:38.135027155Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_rabbitmqdata" + ], + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-seaweedfs-1": { + "service": "seaweedfs", + "started_at": "2026-11-12T01:25:38.293452184Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_seaweeddata" + ], + "memory_bytes": 805306368, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-tempo-1": { + "service": "tempo", + "started_at": "2026-11-12T01:25:37.986295937Z", + "running": true, + "health": null, + "volumes": [ + "mtusi-lab6_tempodata" + ], + "memory_bytes": 1073741824, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-valkey-1": { + "service": "valkey", + "started_at": "2026-11-12T01:25:37.254699807Z", + "running": true, + "health": "healthy", + "volumes": [ + "mtusi-lab6_valkeydata" + ], + "memory_bytes": 134217728, + "nano_cpus": 500000000 + }, + "mtusi-lab6-worker-1": { + "service": "worker", + "started_at": "2026-11-12T01:25:42.656281441Z", + "running": true, + "health": "healthy", + "volumes": [], + "memory_bytes": 805306368, + "nano_cpus": 1000000000 + } + }, + "all_services_restarted": true, + "volumes_preserved": true +} \ No newline at end of file diff --git a/evidence/recheck_2026-11-12/obs_cost.json b/evidence/recheck_2026-11-12/obs_cost.json new file mode 100644 index 0000000..659e144 --- /dev/null +++ b/evidence/recheck_2026-11-12/obs_cost.json @@ -0,0 +1,1105 @@ +{ + "date": "2026-11-12 04:49:48", + "args": { + "out": "evidence\\recheck_2026-11-12", + "profiles": [ + "off", + "on100", + "head10", + "tail" + ], + "merge": false, + "seconds": 30, + "concurrency": 64, + "procs": 8, + "flush": 25, + "repeat": 2 + }, + "faults": "http500:0.02,slow:0.02:0.6", + "results": [ + { + "profile": "off", + "post": { + "requests": 42499, + "ok": 41641, + "errors": { + "500": 858 + }, + "seconds": 30, + "ok_rps": 1388.0, + "error_ratio": 0.0202, + "p50_ms": 16.03, + "p95_ms": 72.03, + "p99_ms": 617.51 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 54.6 + }, + "api": { + "avg_cpu_percent": 49.7, + "peak_mem_mib": 64.0 + }, + "otel-collector": { + "avg_cpu_percent": 0.1, + "peak_mem_mib": 43.7 + }, + "tempo": { + "avg_cpu_percent": 0.3, + "peak_mem_mib": 388.5 + }, + "lb": { + "avg_cpu_percent": 22.6, + "peak_mem_mib": 21.8 + }, + "grafana": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 315.2 + }, + "rabbitmq": { + "avg_cpu_percent": 8.3, + "peak_mem_mib": 178.2 + }, + "postgres": { + "avg_cpu_percent": 94.0, + "peak_mem_mib": 263.0 + }, + "loki": { + "avg_cpu_percent": 0.5, + "peak_mem_mib": 121.8 + }, + "seaweedfs": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 505.8 + }, + "prometheus": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 60.9 + }, + "valkey": { + "avg_cpu_percent": 4.5, + "peak_mem_mib": 10.7 + } + }, + "collector_delta": { + "receiver_accepted_spans": 0, + "receiver_accepted_log_records": 0, + "receiver_accepted_metric_points": 0, + "receiver_refused_spans": 0, + "exporter_sent_spans": 0, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 0, + "loki_bytes_delta": 0, + "errors_injected_500": 873, + "slow_requests_gt500ms": 888, + "traces_in_tempo": { + "all_http": 0, + "error": 0, + "slow_gt500ms": 0 + }, + "kept_ratio": { + "errors": 0.0, + "slow": 0.0 + }, + "rps_runs": [ + 1348.0, + 1388.0 + ], + "p95_runs": [ + 74.05, + 72.03 + ], + "runs": [ + { + "profile": "off", + "post": { + "requests": 41303, + "ok": 40440, + "errors": { + "500": 863 + }, + "seconds": 30, + "ok_rps": 1348.0, + "error_ratio": 0.0209, + "p50_ms": 17.39, + "p95_ms": 74.05, + "p99_ms": 618.63 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.6, + "peak_mem_mib": 54.6 + }, + "api": { + "avg_cpu_percent": 50.6, + "peak_mem_mib": 76.4 + }, + "otel-collector": { + "avg_cpu_percent": 0.1, + "peak_mem_mib": 45.1 + }, + "tempo": { + "avg_cpu_percent": 0.3, + "peak_mem_mib": 426.2 + }, + "lb": { + "avg_cpu_percent": 23.0, + "peak_mem_mib": 18.3 + }, + "grafana": { + "avg_cpu_percent": 1.1, + "peak_mem_mib": 314.4 + }, + "rabbitmq": { + "avg_cpu_percent": 14.5, + "peak_mem_mib": 177.9 + }, + "postgres": { + "avg_cpu_percent": 95.9, + "peak_mem_mib": 264.3 + }, + "loki": { + "avg_cpu_percent": 0.5, + "peak_mem_mib": 119.8 + }, + "seaweedfs": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 524.7 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 75.6 + }, + "valkey": { + "avg_cpu_percent": 4.7, + "peak_mem_mib": 10.7 + } + }, + "collector_delta": { + "receiver_accepted_spans": 0, + "receiver_accepted_log_records": 0, + "receiver_accepted_metric_points": 0, + "receiver_refused_spans": 0, + "exporter_sent_spans": 0, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 0, + "loki_bytes_delta": 0, + "errors_injected_500": 873, + "slow_requests_gt500ms": 823, + "traces_in_tempo": { + "all_http": 0, + "error": 0, + "slow_gt500ms": 0 + }, + "kept_ratio": { + "errors": 0.0, + "slow": 0.0 + } + }, + { + "profile": "off", + "post": { + "requests": 42499, + "ok": 41641, + "errors": { + "500": 858 + }, + "seconds": 30, + "ok_rps": 1388.0, + "error_ratio": 0.0202, + "p50_ms": 16.03, + "p95_ms": 72.03, + "p99_ms": 617.51 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 54.6 + }, + "api": { + "avg_cpu_percent": 49.7, + "peak_mem_mib": 64.0 + }, + "otel-collector": { + "avg_cpu_percent": 0.1, + "peak_mem_mib": 43.7 + }, + "tempo": { + "avg_cpu_percent": 0.3, + "peak_mem_mib": 388.5 + }, + "lb": { + "avg_cpu_percent": 22.6, + "peak_mem_mib": 21.8 + }, + "grafana": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 315.2 + }, + "rabbitmq": { + "avg_cpu_percent": 8.3, + "peak_mem_mib": 178.2 + }, + "postgres": { + "avg_cpu_percent": 94.0, + "peak_mem_mib": 263.0 + }, + "loki": { + "avg_cpu_percent": 0.5, + "peak_mem_mib": 121.8 + }, + "seaweedfs": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 505.8 + }, + "prometheus": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 60.9 + }, + "valkey": { + "avg_cpu_percent": 4.5, + "peak_mem_mib": 10.7 + } + }, + "collector_delta": { + "receiver_accepted_spans": 0, + "receiver_accepted_log_records": 0, + "receiver_accepted_metric_points": 0, + "receiver_refused_spans": 0, + "exporter_sent_spans": 0, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 0, + "loki_bytes_delta": 0, + "errors_injected_500": 873, + "slow_requests_gt500ms": 888, + "traces_in_tempo": { + "all_http": 0, + "error": 0, + "slow_gt500ms": 0 + }, + "kept_ratio": { + "errors": 0.0, + "slow": 0.0 + } + } + ] + }, + { + "profile": "on100", + "post": { + "requests": 37970, + "ok": 37178, + "errors": { + "500": 792 + }, + "seconds": 30, + "ok_rps": 1239.3, + "error_ratio": 0.0209, + "p50_ms": 24.47, + "p95_ms": 102.73, + "p99_ms": 627.88 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.7, + "peak_mem_mib": 60.3 + }, + "api": { + "avg_cpu_percent": 78.4, + "peak_mem_mib": 75.4 + }, + "otel-collector": { + "avg_cpu_percent": 3.3, + "peak_mem_mib": 72.0 + }, + "tempo": { + "avg_cpu_percent": 7.9, + "peak_mem_mib": 526.0 + }, + "lb": { + "avg_cpu_percent": 19.5, + "peak_mem_mib": 21.7 + }, + "grafana": { + "avg_cpu_percent": 0.8, + "peak_mem_mib": 321.6 + }, + "rabbitmq": { + "avg_cpu_percent": 12.0, + "peak_mem_mib": 188.4 + }, + "postgres": { + "avg_cpu_percent": 89.5, + "peak_mem_mib": 284.4 + }, + "loki": { + "avg_cpu_percent": 4.8, + "peak_mem_mib": 216.5 + }, + "seaweedfs": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 494.4 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 61.0 + }, + "valkey": { + "avg_cpu_percent": 5.0, + "peak_mem_mib": 11.5 + } + }, + "collector_delta": { + "receiver_accepted_spans": 89310, + "receiver_accepted_log_records": 43824, + "receiver_accepted_metric_points": 396, + "receiver_refused_spans": 0, + "exporter_sent_spans": 89310, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 22323200, + "loki_bytes_delta": 30490624, + "errors_injected_500": 800, + "slow_requests_gt500ms": 737, + "traces_in_tempo": { + "all_http": 5000, + "error": 881, + "slow_gt500ms": 850 + }, + "kept_ratio": { + "errors": 1.101, + "slow": 1.153 + }, + "rps_runs": [ + 1251.0, + 1239.3 + ], + "p95_runs": [ + 97.91, + 102.73 + ], + "runs": [ + { + "profile": "on100", + "post": { + "requests": 38267, + "ok": 37530, + "errors": { + "500": 737 + }, + "seconds": 30, + "ok_rps": 1251.0, + "error_ratio": 0.0193, + "p50_ms": 24.79, + "p95_ms": 97.91, + "p99_ms": 631.54 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 2.2, + "peak_mem_mib": 64.5 + }, + "api": { + "avg_cpu_percent": 80.6, + "peak_mem_mib": 83.2 + }, + "otel-collector": { + "avg_cpu_percent": 3.4, + "peak_mem_mib": 75.0 + }, + "tempo": { + "avg_cpu_percent": 11.2, + "peak_mem_mib": 745.7 + }, + "lb": { + "avg_cpu_percent": 20.2, + "peak_mem_mib": 21.5 + }, + "grafana": { + "avg_cpu_percent": 1.1, + "peak_mem_mib": 320.0 + }, + "rabbitmq": { + "avg_cpu_percent": 12.1, + "peak_mem_mib": 157.0 + }, + "postgres": { + "avg_cpu_percent": 92.6, + "peak_mem_mib": 282.2 + }, + "loki": { + "avg_cpu_percent": 4.4, + "peak_mem_mib": 194.8 + }, + "seaweedfs": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 494.7 + }, + "prometheus": { + "avg_cpu_percent": 0.5, + "peak_mem_mib": 60.8 + }, + "valkey": { + "avg_cpu_percent": 5.4, + "peak_mem_mib": 11.5 + } + }, + "collector_delta": { + "receiver_accepted_spans": 90462, + "receiver_accepted_log_records": 44825, + "receiver_accepted_metric_points": 396, + "receiver_refused_spans": 0, + "exporter_sent_spans": 90462, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 47091712, + "loki_bytes_delta": 30777344, + "errors_injected_500": 761, + "slow_requests_gt500ms": 808, + "traces_in_tempo": { + "all_http": 5000, + "error": 848, + "slow_gt500ms": 902 + }, + "kept_ratio": { + "errors": 1.114, + "slow": 1.116 + } + }, + { + "profile": "on100", + "post": { + "requests": 37970, + "ok": 37178, + "errors": { + "500": 792 + }, + "seconds": 30, + "ok_rps": 1239.3, + "error_ratio": 0.0209, + "p50_ms": 24.47, + "p95_ms": 102.73, + "p99_ms": 627.88 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.7, + "peak_mem_mib": 60.3 + }, + "api": { + "avg_cpu_percent": 78.4, + "peak_mem_mib": 75.4 + }, + "otel-collector": { + "avg_cpu_percent": 3.3, + "peak_mem_mib": 72.0 + }, + "tempo": { + "avg_cpu_percent": 7.9, + "peak_mem_mib": 526.0 + }, + "lb": { + "avg_cpu_percent": 19.5, + "peak_mem_mib": 21.7 + }, + "grafana": { + "avg_cpu_percent": 0.8, + "peak_mem_mib": 321.6 + }, + "rabbitmq": { + "avg_cpu_percent": 12.0, + "peak_mem_mib": 188.4 + }, + "postgres": { + "avg_cpu_percent": 89.5, + "peak_mem_mib": 284.4 + }, + "loki": { + "avg_cpu_percent": 4.8, + "peak_mem_mib": 216.5 + }, + "seaweedfs": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 494.4 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 61.0 + }, + "valkey": { + "avg_cpu_percent": 5.0, + "peak_mem_mib": 11.5 + } + }, + "collector_delta": { + "receiver_accepted_spans": 89310, + "receiver_accepted_log_records": 43824, + "receiver_accepted_metric_points": 396, + "receiver_refused_spans": 0, + "exporter_sent_spans": 89310, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 22323200, + "loki_bytes_delta": 30490624, + "errors_injected_500": 800, + "slow_requests_gt500ms": 737, + "traces_in_tempo": { + "all_http": 5000, + "error": 881, + "slow_gt500ms": 850 + }, + "kept_ratio": { + "errors": 1.101, + "slow": 1.153 + } + } + ] + }, + { + "profile": "head10", + "post": { + "requests": 39840, + "ok": 39078, + "errors": { + "500": 762 + }, + "seconds": 30, + "ok_rps": 1302.6, + "error_ratio": 0.0191, + "p50_ms": 21.48, + "p95_ms": 79.8, + "p99_ms": 622.79 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 60.2 + }, + "api": { + "avg_cpu_percent": 69.4, + "peak_mem_mib": 74.0 + }, + "otel-collector": { + "avg_cpu_percent": 1.9, + "peak_mem_mib": 68.9 + }, + "tempo": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 755.1 + }, + "lb": { + "avg_cpu_percent": 19.9, + "peak_mem_mib": 21.7 + }, + "grafana": { + "avg_cpu_percent": 1.5, + "peak_mem_mib": 325.1 + }, + "rabbitmq": { + "avg_cpu_percent": 10.9, + "peak_mem_mib": 194.5 + }, + "postgres": { + "avg_cpu_percent": 91.7, + "peak_mem_mib": 303.6 + }, + "loki": { + "avg_cpu_percent": 4.6, + "peak_mem_mib": 281.0 + }, + "seaweedfs": { + "avg_cpu_percent": 1.2, + "peak_mem_mib": 494.4 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 61.0 + }, + "valkey": { + "avg_cpu_percent": 5.0, + "peak_mem_mib": 11.4 + } + }, + "collector_delta": { + "receiver_accepted_spans": 9291, + "receiver_accepted_log_records": 45766, + "receiver_accepted_metric_points": 396, + "receiver_refused_spans": 0, + "exporter_sent_spans": 9458, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 32768, + "loki_bytes_delta": 32833536, + "errors_injected_500": 774, + "slow_requests_gt500ms": 802, + "traces_in_tempo": { + "all_http": 4411, + "error": 81, + "slow_gt500ms": 93 + }, + "kept_ratio": { + "errors": 0.105, + "slow": 0.116 + }, + "rps_runs": [ + 1136.7, + 1302.6 + ], + "p95_runs": [ + 123.14, + 79.8 + ], + "runs": [ + { + "profile": "head10", + "post": { + "requests": 34821, + "ok": 34102, + "errors": { + "500": 719 + }, + "seconds": 30, + "ok_rps": 1136.7, + "error_ratio": 0.0206, + "p50_ms": 26.35, + "p95_ms": 123.14, + "p99_ms": 630.88 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 2.0, + "peak_mem_mib": 62.5 + }, + "api": { + "avg_cpu_percent": 67.6, + "peak_mem_mib": 83.7 + }, + "otel-collector": { + "avg_cpu_percent": 1.9, + "peak_mem_mib": 68.4 + }, + "tempo": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 472.9 + }, + "lb": { + "avg_cpu_percent": 19.2, + "peak_mem_mib": 23.2 + }, + "grafana": { + "avg_cpu_percent": 0.9, + "peak_mem_mib": 322.9 + }, + "rabbitmq": { + "avg_cpu_percent": 10.5, + "peak_mem_mib": 194.3 + }, + "postgres": { + "avg_cpu_percent": 89.7, + "peak_mem_mib": 302.1 + }, + "loki": { + "avg_cpu_percent": 4.8, + "peak_mem_mib": 239.9 + }, + "seaweedfs": { + "avg_cpu_percent": 1.5, + "peak_mem_mib": 494.4 + }, + "prometheus": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 60.9 + }, + "valkey": { + "avg_cpu_percent": 5.1, + "peak_mem_mib": 12.8 + } + }, + "collector_delta": { + "receiver_accepted_spans": 8506, + "receiver_accepted_log_records": 40586, + "receiver_accepted_metric_points": 396, + "receiver_refused_spans": 0, + "exporter_sent_spans": 8506, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 1200128, + "loki_bytes_delta": 4431872, + "errors_injected_500": 727, + "slow_requests_gt500ms": 671, + "traces_in_tempo": { + "all_http": 3919, + "error": 89, + "slow_gt500ms": 80 + }, + "kept_ratio": { + "errors": 0.122, + "slow": 0.119 + } + }, + { + "profile": "head10", + "post": { + "requests": 39840, + "ok": 39078, + "errors": { + "500": 762 + }, + "seconds": 30, + "ok_rps": 1302.6, + "error_ratio": 0.0191, + "p50_ms": 21.48, + "p95_ms": 79.8, + "p99_ms": 622.79 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 60.2 + }, + "api": { + "avg_cpu_percent": 69.4, + "peak_mem_mib": 74.0 + }, + "otel-collector": { + "avg_cpu_percent": 1.9, + "peak_mem_mib": 68.9 + }, + "tempo": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 755.1 + }, + "lb": { + "avg_cpu_percent": 19.9, + "peak_mem_mib": 21.7 + }, + "grafana": { + "avg_cpu_percent": 1.5, + "peak_mem_mib": 325.1 + }, + "rabbitmq": { + "avg_cpu_percent": 10.9, + "peak_mem_mib": 194.5 + }, + "postgres": { + "avg_cpu_percent": 91.7, + "peak_mem_mib": 303.6 + }, + "loki": { + "avg_cpu_percent": 4.6, + "peak_mem_mib": 281.0 + }, + "seaweedfs": { + "avg_cpu_percent": 1.2, + "peak_mem_mib": 494.4 + }, + "prometheus": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 61.0 + }, + "valkey": { + "avg_cpu_percent": 5.0, + "peak_mem_mib": 11.4 + } + }, + "collector_delta": { + "receiver_accepted_spans": 9291, + "receiver_accepted_log_records": 45766, + "receiver_accepted_metric_points": 396, + "receiver_refused_spans": 0, + "exporter_sent_spans": 9458, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 32768, + "loki_bytes_delta": 32833536, + "errors_injected_500": 774, + "slow_requests_gt500ms": 802, + "traces_in_tempo": { + "all_http": 4411, + "error": 81, + "slow_gt500ms": 93 + }, + "kept_ratio": { + "errors": 0.105, + "slow": 0.116 + } + } + ] + }, + { + "profile": "tail", + "post": { + "requests": 39821, + "ok": 38989, + "errors": { + "500": 832 + }, + "seconds": 30, + "ok_rps": 1299.6, + "error_ratio": 0.0209, + "p50_ms": 23.35, + "p95_ms": 93.86, + "p99_ms": 626.16 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 64.2 + }, + "api": { + "avg_cpu_percent": 77.7, + "peak_mem_mib": 75.5 + }, + "otel-collector": { + "avg_cpu_percent": 3.4, + "peak_mem_mib": 108.4 + }, + "tempo": { + "avg_cpu_percent": 1.9, + "peak_mem_mib": 443.0 + }, + "lb": { + "avg_cpu_percent": 18.6, + "peak_mem_mib": 21.6 + }, + "grafana": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 325.1 + }, + "rabbitmq": { + "avg_cpu_percent": 9.5, + "peak_mem_mib": 175.5 + }, + "postgres": { + "avg_cpu_percent": 88.4, + "peak_mem_mib": 303.5 + }, + "loki": { + "avg_cpu_percent": 4.3, + "peak_mem_mib": 317.3 + }, + "seaweedfs": { + "avg_cpu_percent": 4.2, + "peak_mem_mib": 692.3 + }, + "prometheus": { + "avg_cpu_percent": 0.5, + "peak_mem_mib": 61.1 + }, + "valkey": { + "avg_cpu_percent": 4.9, + "peak_mem_mib": 11.7 + } + }, + "collector_delta": { + "receiver_accepted_spans": 92273, + "receiver_accepted_log_records": 45427, + "receiver_accepted_metric_points": 396, + "receiver_refused_spans": 0, + "exporter_sent_spans": 13252, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 1703936, + "loki_bytes_delta": 32804864, + "errors_injected_500": 837, + "slow_requests_gt500ms": 743, + "traces_in_tempo": { + "all_http": 5000, + "error": 925, + "slow_gt500ms": 834 + }, + "kept_ratio": { + "errors": 1.105, + "slow": 1.122 + }, + "rps_runs": [ + 1282.3, + 1299.6 + ], + "p95_runs": [ + 86.59, + 93.86 + ], + "runs": [ + { + "profile": "tail", + "post": { + "requests": 39275, + "ok": 38469, + "errors": { + "500": 806 + }, + "seconds": 30, + "ok_rps": 1282.3, + "error_ratio": 0.0205, + "p50_ms": 23.74, + "p95_ms": 86.59, + "p99_ms": 631.13 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 2.1, + "peak_mem_mib": 59.8 + }, + "api": { + "avg_cpu_percent": 77.1, + "peak_mem_mib": 75.4 + }, + "otel-collector": { + "avg_cpu_percent": 4.6, + "peak_mem_mib": 105.8 + }, + "tempo": { + "avg_cpu_percent": 0.8, + "peak_mem_mib": 719.9 + }, + "lb": { + "avg_cpu_percent": 19.7, + "peak_mem_mib": 22.0 + }, + "grafana": { + "avg_cpu_percent": 1.0, + "peak_mem_mib": 325.2 + }, + "rabbitmq": { + "avg_cpu_percent": 12.5, + "peak_mem_mib": 185.8 + }, + "postgres": { + "avg_cpu_percent": 93.0, + "peak_mem_mib": 302.7 + }, + "loki": { + "avg_cpu_percent": 5.8, + "peak_mem_mib": 312.2 + }, + "seaweedfs": { + "avg_cpu_percent": 1.4, + "peak_mem_mib": 494.4 + }, + "prometheus": { + "avg_cpu_percent": 0.6, + "peak_mem_mib": 60.9 + }, + "valkey": { + "avg_cpu_percent": 5.1, + "peak_mem_mib": 11.4 + } + }, + "collector_delta": { + "receiver_accepted_spans": 91490, + "receiver_accepted_log_records": 45024, + "receiver_accepted_metric_points": 396, + "receiver_refused_spans": 0, + "exporter_sent_spans": 13023, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 401408, + "loki_bytes_delta": 34607104, + "errors_injected_500": 818, + "slow_requests_gt500ms": 793, + "traces_in_tempo": { + "all_http": 5000, + "error": 907, + "slow_gt500ms": 883 + }, + "kept_ratio": { + "errors": 1.109, + "slow": 1.113 + } + }, + { + "profile": "tail", + "post": { + "requests": 39821, + "ok": 38989, + "errors": { + "500": 832 + }, + "seconds": 30, + "ok_rps": 1299.6, + "error_ratio": 0.0209, + "p50_ms": 23.35, + "p95_ms": 93.86, + "p99_ms": 626.16 + }, + "docker_stats": { + "worker": { + "avg_cpu_percent": 1.3, + "peak_mem_mib": 64.2 + }, + "api": { + "avg_cpu_percent": 77.7, + "peak_mem_mib": 75.5 + }, + "otel-collector": { + "avg_cpu_percent": 3.4, + "peak_mem_mib": 108.4 + }, + "tempo": { + "avg_cpu_percent": 1.9, + "peak_mem_mib": 443.0 + }, + "lb": { + "avg_cpu_percent": 18.6, + "peak_mem_mib": 21.6 + }, + "grafana": { + "avg_cpu_percent": 0.7, + "peak_mem_mib": 325.1 + }, + "rabbitmq": { + "avg_cpu_percent": 9.5, + "peak_mem_mib": 175.5 + }, + "postgres": { + "avg_cpu_percent": 88.4, + "peak_mem_mib": 303.5 + }, + "loki": { + "avg_cpu_percent": 4.3, + "peak_mem_mib": 317.3 + }, + "seaweedfs": { + "avg_cpu_percent": 4.2, + "peak_mem_mib": 692.3 + }, + "prometheus": { + "avg_cpu_percent": 0.5, + "peak_mem_mib": 61.1 + }, + "valkey": { + "avg_cpu_percent": 4.9, + "peak_mem_mib": 11.7 + } + }, + "collector_delta": { + "receiver_accepted_spans": 92273, + "receiver_accepted_log_records": 45427, + "receiver_accepted_metric_points": 396, + "receiver_refused_spans": 0, + "exporter_sent_spans": 13252, + "exporter_send_failed_spans": 0 + }, + "tempo_bytes_delta": 1703936, + "loki_bytes_delta": 32804864, + "errors_injected_500": 837, + "slow_requests_gt500ms": 743, + "traces_in_tempo": { + "all_http": 5000, + "error": 925, + "slow_gt500ms": 834 + }, + "kept_ratio": { + "errors": 1.105, + "slow": 1.122 + } + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/recheck_2026-11-12/obs_cost.png b/evidence/recheck_2026-11-12/obs_cost.png new file mode 100644 index 0000000..8a5b65c Binary files /dev/null and b/evidence/recheck_2026-11-12/obs_cost.png differ diff --git a/evidence/recheck_2026-11-12/recovery_checks.txt b/evidence/recheck_2026-11-12/recovery_checks.txt new file mode 100644 index 0000000..a8bd4d0 --- /dev/null +++ b/evidence/recheck_2026-11-12/recovery_checks.txt @@ -0,0 +1,64 @@ +$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +exit=0 + +$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 0, "first_recovery_seconds": 2.344} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +exit=0 + +$ docker compose up -d --wait --no-recreate --no-deps api + Container mtusi-lab6-api-2 Running + Container mtusi-lab6-api-1 Starting + Container mtusi-lab6-api-1 Started + Container mtusi-lab6-api-1 Waiting + Container mtusi-lab6-api-2 Waiting + Container mtusi-lab6-api-1 Healthy + Container mtusi-lab6-api-2 Healthy +exit=0 + +$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 119, "failed_attempts": 0, "first_recovery_seconds": 2.344} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +exit=0 + +$ docker compose up -d --wait --no-recreate --no-deps api + Container mtusi-lab6-api-1 Running + Container mtusi-lab6-api-2 Starting + Container mtusi-lab6-api-2 Started + Container mtusi-lab6-api-1 Waiting + Container mtusi-lab6-api-2 Waiting + Container mtusi-lab6-api-1 Healthy + Container mtusi-lab6-api-2 Healthy +exit=0 + +$ docker compose run --rm -T s3-init + Container mtusi-lab6-seaweedfs-1 Running + Container mtusi-lab6-seaweedfs-1 Waiting + Container mtusi-lab6-seaweedfs-1 Healthy + Container mtusi-lab6-s3-init-run-ed2a1968cae4 Creating + Container mtusi-lab6-s3-init-run-ed2a1968cae4 Created +bucket messenger-attachments уже существует +анонимный доступ запрещён (HTTP 403) +ключ API ограничен одним bucket (create_bucket отклонён: AccessDenied) +bootstrap завершён +exit=0 + +$ docker compose run --rm -T s3-init + Container mtusi-lab6-seaweedfs-1 Running + Container mtusi-lab6-seaweedfs-1 Waiting + Container mtusi-lab6-seaweedfs-1 Healthy + Container mtusi-lab6-s3-init-run-4edec659c8c0 Creating + Container mtusi-lab6-s3-init-run-4edec659c8c0 Created +bucket messenger-attachments уже существует +анонимный доступ запрещён (HTTP 403) +ключ API ограничен одним bucket (create_bucket отклонён: AccessDenied) +bootstrap завершён +exit=0 + +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +exit=0 diff --git a/evidence/recheck_2026-11-12/research_runtime.json b/evidence/recheck_2026-11-12/research_runtime.json new file mode 100644 index 0000000..5fd286e --- /dev/null +++ b/evidence/recheck_2026-11-12/research_runtime.json @@ -0,0 +1,84 @@ +{ + "tempo_discarded_spans_before": null, + "tempo_discarded_spans_after": null, + "containers": { + "mtusi-lab6-api-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 268435456, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-api-2": { + "running": true, + "oom_killed": false, + "memory_bytes": 268435456, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-grafana-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-lb-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 134217728, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-loki-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-otel-collector-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 402653184, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-postgres-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-prometheus-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-rabbitmq-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 536870912, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-seaweedfs-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 805306368, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-tempo-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 1073741824, + "nano_cpus": 1000000000 + }, + "mtusi-lab6-valkey-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 134217728, + "nano_cpus": 500000000 + }, + "mtusi-lab6-worker-1": { + "running": true, + "oom_killed": false, + "memory_bytes": 805306368, + "nano_cpus": 1000000000 + } + } +} \ No newline at end of file diff --git a/evidence/recheck_2026-11-12/source_metadata.json b/evidence/recheck_2026-11-12/source_metadata.json new file mode 100644 index 0000000..70b20a5 --- /dev/null +++ b/evidence/recheck_2026-11-12/source_metadata.json @@ -0,0 +1,89 @@ +{ + "base_commit": "1d908578e3ccdec8440efdf2e5135580170159aa", + "branch": "solution/bpi2302-kushnarenko", + "source_sha256": { + "research/requirements.txt": "0a0f9ffaa9b56d590cb38c77f9da8dbc64f2d503b69fc27b0116b9a9594b2591", + "src/app/s3_recovery.py": "582b56ebd579c94670f28e801b7d66aa6dab2915bcbdc57f129f18651f6aa228", + "student-tests/README.md": "ab6c3c99ef61e14ba30037d16d51b3759d0ee02b71ecbf84b51ae941dfb3b90d", + "student-tests/test_dependencies.py": "e152b87441d1374ce9b69d3d441d262e47890313d2ccc798f5e2cff2e3719399", + "student-tests/test_expiry.py": "16a374ebe3df15c50c07a9fc3bc801c6c532dde3c4185b164c6aea4605923f83", + "student-tests/test_reconcile5.py": "7df51d06fc6da180fada10d3d6e1233c9462870ec18f6d55753a36e30dca458b", + "Dockerfile": "f1dc711cacf00e3e2bd6b4a0616c925e357446ebda370e7e5fdb3170eaecd4cb", + "compose.yaml": "acb6f5bb7a96221a077cff4f5311125cfafb16a98e0478a48c6c47b3b974b77d", + "infra/README.md": "33a4f7b870fd0b2bed98d46daf26d45dd7436176739672925246f8e3fe7c2278", + "infra/grafana/build_dashboard.py": "8a1eb1a037c8aeb473cfce9610682b15c1b92f4ed85efe95e61b8f293c87debf", + "infra/grafana/dashboards/messenger.json": "d897fa6a9ac06b05520906773cce0cba3bb5118576263bc82596d462b9b9ab94", + "infra/grafana/provisioning/dashboards/dashboards.yaml": "f2f11fb3a1d5c78745ededf551c187994bef0d3e0fbadc1c7bae3ae4bd76c52e", + "infra/grafana/provisioning/datasources/datasources.yaml": "fe956b90867757e930ba87022a3e77584b42949ddd814ed058d6502d64200e74", + "infra/haproxy/haproxy.cfg": "511fb24b32740c332cd7b6ea42d3f1cf252e1452de754ff97c8e6886f3692061", + "infra/loki/loki.yaml": "9069174e115badf13e559f2c9e8d3593603b451416737d9fa7d2724113da0bd3", + "infra/otel/README.md": "709332a3e910f9d42cb2a44c8d24b13aff6ce0ac7b671840e3cc273ddb82d3d6", + "infra/otel/collector-tail.yaml": "3e3b5c9a371a2f9f88d4f96a6da11630e652eeea42da7dc8386c7062c7887723", + "infra/otel/collector.yaml": "c0931a19c3b65aa893cff72aa4528aa5bf814b64ffac563506aa0ce59cc52d78", + "infra/prometheus/prometheus.yml": "b3c5272057d171c10f01674cc02e71f62d30c6abd4a560589cb3bc106de66294", + "infra/prometheus/rules/messenger.yml": "370cf706c19eed8acca5e310f3b036c88ef209e30cffde019d124fad35bd3252", + "infra/tempo/tempo.yaml": "f87d6dbc22c25e4007024ade5429a8e05a2176873da69004a8f234baa7877668", + "requirements.txt": "3538783836e3ba370df10d9335409a77fe1576f0269fea1917d54e37926e47a7", + "research/failover_sequence.py": "9d5dec92d067f467dc7842f9c698b39de28fb5c91902668917abaf1cceae5935", + "research/incidents.py": "4b2cd148b3b86e3e85565d4d7a5a11ad0d798f53a1709c077bb89952d4c2a98d", + "research/loadgen.py": "e4bee7ea46ad97822d440ea0bed35e36615ae60158ab4bc770179079a7558303", + "research/obs_cost.py": "88ecc651d0ca5d51eb5e0afafea09f12b4cd3eb91aa3400ef04eff88b502bfd0", + "research/plot_obs_cost.py": "67e491695321a43882d9ae32645914bfe02650ee022858a43049781f3a92c1ba", + "research/queue_overload.py": "6d0a3fdb92899a04fb1d98f5d1f4fbbb27396284dc8186266522b83c85565dbc", + "research/rolling_update.py": "447da4a9e2161bb232669f0558d6cb15ae3f0ef181fdf563e19cd56135b58240", + "scripts/check_contract.py": "61e300fe5c1074360418e24b4653b12f96808247a2beaf2a64859aa360f81493", + "scripts/init-secrets.sh": "3e238e5066f323e424f26cea9e360792d8a3d55d3755742e81d130707bfe967d", + "scripts/restart.cmd": "b4f0ada9f8b0d009a3fbaa84aecef04b6d4d0f0229c938a3a8c68ad3ba76a7f8", + "scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b", + "scripts/restart.sh": "7a567573bf3510a7d4ab5a10f656a9c3ef2f0ead1ae0802f6b4a55b800d235d0", + "scripts/restore-replicas.sh": "4e89f3f27ab5e9c958d3716c3bb2a8323133d773617bfec4b137f1c0b40d1e97", + "scripts/rolling-update.sh": "9663b1fe8eeda5dafdc4338105e77cf7696c117d0f4652950ae8283d50439d4d", + "scripts/stop-one.cmd": "a12b882f0f580aa10591d167e69e01d4a5fc8c8e0520773a542b21b1aac43d39", + "scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811", + "scripts/stop-one.sh": "9ae4a3d75e7d857ac3c62c2d97b95282b33a30b1affbf83df6de8f88c2ffded1", + "scripts/validate_openapi.py": "cd7a33221b1860c5d2330e52afa6cb222d375db671ae9ab379ba51a0d3a77946", + "src/app/attachments.py": "ba4f48e2220e39e4d3dd4d04f571afc1b3bf68f1b809b2eaeff240a1da3b3329", + "src/app/broker.py": "b68f297acd17bbd4d63b140cef06b89f73048da6d35fc6027717d54c5cf3733b", + "src/app/config.py": "f9483cb31afd5a9be0f00e2ea71a32dd1216d5d48b4c0595ee43d31ffbfa0a61", + "src/app/db.py": "fc6a3872f977550b2fef7cd9db9b36e1fdd8b1fc95abe77f36916d2c8422222f", + "src/app/images.py": "3dc7434cbfbc9d4c01b95eb897c1332b2a2748dd9df88c99a7086f3e4cdf4c41", + "src/app/main.py": "1ec3323d1edfcad9a54f8f7b62e8064daaa69555cb5cd2b87da20f7313bae954", + "src/app/migrate.py": "cabc192ca4182af780636415a008d2dd4636dad7d3cf0f4a8a2b52a48aedaa7e", + "src/app/outbox.py": "4255f981c059fad0dee7d355d746f53a3a1c86a8d2e7cbd9aed3e12bedb895f0", + "src/app/passwords.py": "836fc282093058f6717eaa6374775251ed655dcd517f325dc60323a88154f261", + "src/app/pipeline.py": "222ac87f275e4607ecdc98cdc5f68d8957fd7f68603af2d2643f3026445a3fe1", + "src/app/reconcile.py": "c7dd35bf40e672ffe15f0e878913e17f6c6d9de3f69922b4edb38d1802330dc4", + "src/app/replay.py": "2669b9e5fa2f6fb83dbe457f0241db166eb3e370e21d81c3e32f922060f52fa5", + "src/app/s3.py": "86cb37980023d02e56468a355793b69a8c5f80246d36460e17f49c102385f0de", + "src/app/s3_bootstrap.py": "216d8121d6e7b25d95b845f4deaff18fd31eeb28484e51fe4c463c646a3af0d1", + "src/app/serve.py": "3ebe12cf8cb2dddfc7d7234bba9bd190d868cc88d02e61fe640ba51dcf115b55", + "src/app/sessions.py": "f975eb27c50c024f3e562f755ee97bb7bfe21cf5e01cc1d347bf107971c30670", + "src/app/telemetry.py": "16a789f79fa185598d9de9915d89e6b5275fdec819c4df78a092a7c47e952686", + "src/app/web.py": "0c5aa4ab059a1dac64733217e143c9f96fce1f321d2727cf5580b62e04941bfd", + "src/app/worker.py": "23024debdd22076e586d5490874de0de6167e21e472e0a83fbbc86f001e53570", + "student-tests/fixtures/alpha.png": "1f9d6ee26fa7beb4ba179c7c38ae97f5f31e95ad9d4108267352225e6705882f", + "student-tests/fixtures/exif_rot6.jpg": "62386265960512fd51279e40723331c6c52f81ba2325d82afc3fb14f3b332ddc", + "student-tests/fixtures/poison.png": "2be4bea6355e83da4733b1d7f59dc985dfe22cf5d8283eea9b20871d1f43c252", + "student-tests/fixtures/sample.jpg": "33a614b40fe6ae25fdbd99af22e7d9d3e5dbea455b36aac750de112c034a6311", + "student-tests/fixtures/sample.png": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "student-tests/fixtures/small.png": "001e6b4538ce86cbb0c88a34a2b88c111336b8dc8825af81e0f7d9630b731e32", + "student-tests/test_api.py": "2635113842a5b26ec1fdd98ea42fc861edb1d10895e8be6c2eff46ed73fdba9d", + "student-tests/test_attachments.py": "441dc4c5a2d212c135a7d4326a127c95d20bdce6107a8e58fd7d3551415d59c1", + "student-tests/test_idempotency.py": "4ee985a3ef26fefa816fdb136fa2b0644cca13d73fe3a226d1ba975a55698b5b", + "student-tests/test_observability.py": "de61bdb52ca55c361b2394ea82e35039eca38509d3508d9918c4307ddc2336d7", + "student-tests/test_observability_retry.py": "e38e8ab3528479ff3570af1959896571a15d18cfdf8f3761703bd6640feac999", + "student-tests/test_pipeline.py": "1f6b031b0b67a81d4db1e09e8e334efa65a123cefd49c2fd8db05e9505834ad0", + "student-tests/test_resilience5.py": "8906a99d9b3dbc0e47f2e963e95c5ef3c3aa7577767b6c3575eec4617ec7e23b", + "student-tests/test_topology.py": "4401a2db10c65c41430cc0c47c37bee6aefc902880735fb489acad7ad0ff779f" + }, + "hardware": { + "cpu": "Intel Core i9-12900KS", + "cores": 16, + "logical_cpus": 24, + "ram_bytes": 102830256128, + "docker_cpu_count": 24, + "docker_memory_bytes": 50392047616, + "background_other_docker_project": true + }, + "client_python": "3.11.9 (tags/v3.11.9:de54cf5, Apr 2 2024, 10:12:12) [MSC v.1938 64 bit (AMD64)]" +} \ No newline at end of file diff --git a/infra/grafana/build_dashboard.py b/infra/grafana/build_dashboard.py new file mode 100644 index 0000000..83724e5 --- /dev/null +++ b/infra/grafana/build_dashboard.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Генератор dashboard JSON (infra/grafana/dashboards/messenger.json). Запуск: python infra/grafana/build_dashboard.py +Dashboard хранится в Git как JSON и импортируется provisioning-ом автоматически; генератор — только удобство правки.""" +import json +import os + +PROM = {"type": "prometheus", "uid": "prometheus"} +LOKI = {"type": "loki", "uid": "loki"} +TEMPO = {"type": "tempo", "uid": "tempo"} +panel_id = [0] + + +def _id(): + panel_id[0] += 1 + return panel_id[0] + + +def stat_or_ts(title, exprs, x, y, w=8, h=7, unit="short", kind="timeseries", desc="", thresholds=None, ds=PROM, legends=None): + p = {"id": _id(), "type": kind, "title": title, "description": desc, "datasource": ds, + "gridPos": {"x": x, "y": y, "w": w, "h": h}, + "targets": [{"refId": chr(65 + i), "expr": e, "datasource": ds, "legendFormat": (legends or [""] * len(exprs))[i]} + for i, e in enumerate(exprs)], + "fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10}}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}} + if thresholds: + p["fieldConfig"]["defaults"]["thresholds"] = {"mode": "absolute", "steps": thresholds} + return p + + +def row(title, y): + return {"id": _id(), "type": "row", "title": title, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, "collapsed": False, "panels": []} + + +NB = '{http_route!~"/health.*"}' +panels = [] +y = 0 +panels.append(row("API (бизнес-запросы без /health) — запросы, ошибки, задержки", y)); y += 1 +panels += [ + stat_or_ts("RPS по маршрутам", ['sum by (http_route) (rate(http_server_request_duration_seconds_count%s[1m]))' % NB], 0, y, + unit="reqps", legends=["{{http_route}}"], desc="Единицы: запросов в секунду. Источник: http_server_request_duration_seconds_count."), + stat_or_ts("Доля ошибок 5xx (SLI)", ['sli:http_error_ratio:1m'], 8, y, unit="percentunit", + desc="Числитель: 5xx; знаменатель: все запросы кроме /health. 4xx (в т.ч. неверный пароль) не считаются отказом сервиса.", + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 0.05}]), + stat_or_ts("Latency p50 / p95 / p99", [ + 'histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket%s[1m])))' % NB, + 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket%s[1m])))' % NB, + 'histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket%s[1m])))' % NB], + 16, y, unit="s", legends=["p50", "p95", "p99"], + desc="Процентили считаются из суммарной гистограммы всех реплик (sum by le), а не усреднением p95 реплик."), +] +y += 7 +panels += [ + stat_or_ts("RPS по статус-кодам", ['sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count%s[1m]))' % NB], + 0, y, unit="reqps", legends=["{{http_response_status_code}}"]), + stat_or_ts("Latency p95 по маршрутам", ['histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket%s[1m])))' % NB], + 8, y, unit="s", legends=["{{http_route}}"]), + stat_or_ts("DB pool: занято / простаивает / ожидают", ['sum(db_client_connections_usage{state="used"})', 'sum(db_client_connections_usage{state="idle"})', + 'sum(db_client_connections_waiting)'], 16, y, unit="short", + legends=["used", "idle", "waiting"], desc="Пул соединений PostgreSQL по всем репликам API. Рост waiting — насыщение пула."), +] +y += 7 +panels.append(row("Задания (worker) и очередь", y)); y += 1 +panels += [ + stat_or_ts("Задания по исходам", ['sum by (outcome) (rate(messenger_jobs_processed_total[1m]))'], 0, y, unit="ops", legends=["{{outcome}}"], + desc="ready — успешно; failed — финальная ошибка; retried — временная ошибка, запланирован повтор; noop — повторная доставка."), + stat_or_ts("Длительность обработки p50 / p95 (worker)", [ + 'histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))', + 'histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))'], 8, y, unit="s", legends=["p50", "p95"]), + stat_or_ts("Время до ready p50 / p95", [ + 'histogram_quantile(0.50, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[1m])))', + 'histogram_quantile(0.95, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[1m])))'], 16, y, unit="s", legends=["p50", "p95"], + desc="От устойчивого приёма загрузки (202) до ready. Включает ожидание в очереди."), +] +y += 7 +panels += [ + stat_or_ts("Глубина очереди (queued + processing)", ['max(messenger_queue_depth)'], 0, y, unit="short", legends=["jobs"]), + stat_or_ts("Возраст старейшей задачи", ['max(messenger_queue_oldest_job_age_seconds)'], 8, y, unit="s", legends=["oldest"], + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 30}], + desc="Backlog — не задержка обработки: показывает, как долго ждёт самое старое задание."), + stat_or_ts("Worker: занято / ёмкость", ['sum(messenger_worker_busy)', 'sum(messenger_worker_capacity)'], 16, y, legends=["busy", "capacity"], + desc="Насыщение worker: busy == capacity при растущей очереди означает нехватку мощности."), +] +y += 7 +panels.append(row("Ресурсы процессов", y)); y += 1 +panels += [ + stat_or_ts("RSS по сервисам", ['sum by (service_name) (process_memory_rss_bytes)'], 0, y, unit="bytes", legends=["{{service_name}}"]), + stat_or_ts("CPU по сервисам (ядра)", ['sum by (service_name) (rate(process_cpu_time_seconds_total[1m]))'], 8, y, unit="short", legends=["{{service_name}}"], + desc="rate накопленного CPU-времени процесса: 1.0 = одно ядро."), + stat_or_ts("Collector: очередь экспорта и отказы", ['sum by (exporter) (otelcol_exporter_queue_size)', + 'sum by (exporter) (rate(otelcol_exporter_send_failed_spans[1m]))', + 'sum(rate(otelcol_processor_memory_limiter_refused_spans[1m]))'], 16, y, legends=["queue {{exporter}}", "failed spans {{exporter}}", "refused (memory_limiter)"], + desc="Поведение Collector при недоступном backend: ограниченная очередь, отказы, memory_limiter."), +] +y += 7 +panels.append(row("Логи и алерты", y)); y += 1 +panels.append({"id": _id(), "type": "logs", "title": "Структурные логи (API и worker), ошибки и предупреждения", "datasource": LOKI, + "gridPos": {"x": 0, "y": y, "w": 16, "h": 10}, + "targets": [{"refId": "A", "datasource": LOKI, "expr": '{service_name=~"messenger-.*"} | detected_level=~"warn|warning|error|ERROR|WARN"'}], + "options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending"}}) +panels.append({"id": _id(), "type": "alertlist", "title": "Алерты (firing / pending)", "datasource": PROM, + "gridPos": {"x": 16, "y": y, "w": 8, "h": 10}, + "options": {"alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": False, "groupBy": [], "groupMode": "default", + "maxItems": 20, "sortOrder": 1, "stateFilter": {"firing": True, "pending": True, "noData": True, "normal": False, "error": True}}}) +y += 10 +panels.append({"id": _id(), "type": "traces", "title": "Последние трассы API (Tempo)", "datasource": TEMPO, + "gridPos": {"x": 0, "y": y, "w": 24, "h": 8}, + "targets": [{"refId": "A", "datasource": TEMPO, "queryType": "traceql", "query": '{ resource.service.name = "messenger-api" }', "limit": 20}]}) + +dashboard = {"uid": "messenger-overview", "title": "Messenger: API, очередь, worker", "tags": ["messenger", "lab6"], "timezone": "browser", + "schemaVersion": 39, "version": 1, "refresh": "5s", "time": {"from": "now-15m", "to": "now"}, "panels": panels, + "annotations": {"list": []}, "templating": {"list": []}} +out = os.path.join(os.path.dirname(os.path.abspath(__file__)), "dashboards", "messenger.json") +with open(out, "w", encoding="utf-8") as fh: + json.dump(dashboard, fh, ensure_ascii=False, indent=1) +print("saved", out) diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..d7e9693 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,917 @@ +{ + "uid": "messenger-overview", + "title": "Messenger: API, очередь, worker", + "tags": [ + "messenger", + "lab6" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "API (бизнес-запросы без /health) — запросы, ошибки, задержки", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "timeseries", + "title": "RPS по маршрутам", + "description": "Единицы: запросов в секунду. Источник: http_server_request_duration_seconds_count.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 1, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{http_route!~\"/health.*\"}[1m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "{{http_route}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Доля ошибок 5xx (SLI)", + "description": "Числитель: 5xx; знаменатель: все запросы кроме /health. 4xx (в т.ч. неверный пароль) не считаются отказом сервиса.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 1, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "sli:http_error_ratio:1m", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 4, + "type": "timeseries", + "title": "Latency p50 / p95 / p99", + "description": "Процентили считаются из суммарной гистограммы всех реплик (sum by le), а не усреднением p95 реплик.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 1, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{http_route!~\"/health.*\"}[1m])))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{http_route!~\"/health.*\"}[1m])))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "p95" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{http_route!~\"/health.*\"}[1m])))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "RPS по статус-кодам", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{http_route!~\"/health.*\"}[1m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "{{http_response_status_code}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Latency p95 по маршрутам", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 8, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{http_route!~\"/health.*\"}[1m])))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "{{http_route}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "DB pool: занято / простаивает / ожидают", + "description": "Пул соединений PostgreSQL по всем репликам API. Рост waiting — насыщение пула.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 8, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(db_client_connections_usage{state=\"used\"})", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "used" + }, + { + "refId": "B", + "expr": "sum(db_client_connections_usage{state=\"idle\"})", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "idle" + }, + { + "refId": "C", + "expr": "sum(db_client_connections_waiting)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "waiting" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 8, + "type": "row", + "title": "Задания (worker) и очередь", + "gridPos": { + "x": 0, + "y": 15, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 9, + "type": "timeseries", + "title": "Задания по исходам", + "description": "ready — успешно; failed — финальная ошибка; retried — временная ошибка, запланирован повтор; noop — повторная доставка.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (rate(messenger_jobs_processed_total[1m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Длительность обработки p50 / p95 (worker)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 16, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Время до ready p50 / p95", + "description": "От устойчивого приёма загрузки (202) до ready. Включает ожидание в очереди.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 16, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[1m])))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[1m])))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Глубина очереди (queued + processing)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 23, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_depth)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "jobs" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Возраст старейшей задачи", + "description": "Backlog — не задержка обработки: показывает, как долго ждёт самое старое задание.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 23, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_oldest_job_age_seconds)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "oldest" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Worker: занято / ёмкость", + "description": "Насыщение worker: busy == capacity при растущей очереди означает нехватку мощности.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 23, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_worker_busy)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "busy" + }, + { + "refId": "B", + "expr": "sum(messenger_worker_capacity)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "capacity" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 15, + "type": "row", + "title": "Ресурсы процессов", + "gridPos": { + "x": 0, + "y": 30, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 16, + "type": "timeseries", + "title": "RSS по сервисам", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 31, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name) (process_memory_rss_bytes)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "{{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 17, + "type": "timeseries", + "title": "CPU по сервисам (ядра)", + "description": "rate накопленного CPU-времени процесса: 1.0 = одно ядро.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 31, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name) (rate(process_cpu_time_seconds_total[1m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "{{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 18, + "type": "timeseries", + "title": "Collector: очередь экспорта и отказы", + "description": "Поведение Collector при недоступном backend: ограниченная очередь, отказы, memory_limiter.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 31, + "w": 8, + "h": 7 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (exporter) (otelcol_exporter_queue_size)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "queue {{exporter}}" + }, + { + "refId": "B", + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[1m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "failed spans {{exporter}}" + }, + { + "refId": "C", + "expr": "sum(rate(otelcol_processor_memory_limiter_refused_spans[1m]))", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "legendFormat": "refused (memory_limiter)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 19, + "type": "row", + "title": "Логи и алерты", + "gridPos": { + "x": 0, + "y": 38, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 20, + "type": "logs", + "title": "Структурные логи (API и worker), ошибки и предупреждения", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 39, + "w": 16, + "h": 10 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service_name=~\"messenger-.*\"} | detected_level=~\"warn|warning|error|ERROR|WARN\"" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending" + } + }, + { + "id": 21, + "type": "alertlist", + "title": "Алерты (firing / pending)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 39, + "w": 8, + "h": 10 + }, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "firing": true, + "pending": true, + "noData": true, + "normal": false, + "error": true + } + } + }, + { + "id": 22, + "type": "traces", + "title": "Последние трассы API (Tempo)", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "x": 0, + "y": 49, + "w": 24, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{ resource.service.name = \"messenger-api\" }", + "limit": 20 + } + ] + } + ], + "annotations": { + "list": [] + }, + "templating": { + "list": [] + } +} \ No newline at end of file diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..0ae78c0 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 +providers: + - name: messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + allowUiUpdates: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..624153f --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,41 @@ +apiVersion: 1 +# Источники данных Grafana: Prometheus (метрики), Loki (логи), Tempo (трассы). Связи: лог -> трасса (derivedFields), +# трасса -> логи (tracesToLogsV2), трасса -> метрики. +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 5s + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: trace_id + matcherType: label + matcherRegex: trace_id + url: "$${__value.raw}" + datasourceUid: tempo + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: "-1m" + spanEndTimeShift: "1m" + filterByTraceID: true + customQuery: true + query: '{service_name="$${__span.tags["service.name"]}"} | trace_id="$${__span.traceId}"' + tracesToMetrics: + datasourceUid: prometheus + serviceMap: + datasourceUid: prometheus diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 0000000..e446fe8 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,62 @@ +# Балансировщик lab3: HAProxy 3.0. Единственная публичная точка входа; API-реплики наружу не опубликованы. +global + log stdout format raw local0 notice + maxconn 4096 + +defaults + mode http + log global + option dontlognull + option forwardfor + # connect короткий: недоступная реплика быстро заменяется другой (redispatch) + timeout connect 1s + timeout client 30s + timeout server 30s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 5s + retries 3 + option redispatch 1 + # Only retry before a connection is established: upload POST has no idempotency key. + retry-on conn-failure + +# Docker DNS (127.0.0.11): адреса реплик API обновляются без перезапуска LB (scale up/down, пересоздание контейнеров). +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold nx 2s + hold other 2s + hold obsolete 2s + +frontend public + bind :8080 + # If the shared DB is down, live API processes still produce contract JSON, + # validate missing credentials, and serve /health/live independently of DB. + use_backend api_live if { nbsrv(api) eq 0 } + default_backend api + +# Внутренний healthcheck самого LB (контейнерный HEALTHCHECK); наружу не публикуется. +frontend lb_health + bind :8081 + monitor-uri /healthz + +backend api + balance roundrobin + # Проверка готовности каждые 1 с; 2 неудачи подряд -> сервер исключён (обнаружение отказа <= ~3 с), 2 успеха -> возвращён. + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none + server-template api 1-8 api:8080 + +backend api_live + balance roundrobin + option httpchk + http-check send meth GET uri /health/live + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none + server-template api 1-8 api:8080 diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..46b02d6 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,37 @@ +# Loki: хранилище логов (filesystem, volume lokidata). Принимает OTLP/HTTP от Collector (/otlp/v1/logs). +auth_enabled: false +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: 2024-04-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 168h + reject_old_samples: false + volume_enabled: true +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem +analytics: + reporting_enabled: false diff --git a/infra/otel/collector-tail.yaml b/infra/otel/collector-tail.yaml new file mode 100644 index 0000000..a75261d --- /dev/null +++ b/infra/otel/collector-tail.yaml @@ -0,0 +1,64 @@ +# Профиль «tail sampling» (трек 1, lab6): SDK отправляет 100 % спанов, Collector оставляет +# - все трассы с ошибками, - все медленные (> 500 мс), - 10 % остальных. +# Метрики и логи без изменений. Подключается через COLLECTOR_CONFIG=/etc/otelcol/collector-tail.yaml. +receivers: + otlp: + protocols: + grpc: {endpoint: 0.0.0.0:4317} + http: {endpoint: 0.0.0.0:4318} + +processors: + memory_limiter: {check_interval: 1s, limit_mib: 280, spike_limit_mib: 60} + batch: {send_batch_size: 1024, send_batch_max_size: 2048, timeout: 2s} + attributes/scrub: + actions: + - {key: http.request.header.authorization, action: delete} + - {key: http.request.header.cookie, action: delete} + - {key: db.statement, action: delete} + - {key: url.query, action: delete} + tail_sampling: + decision_wait: 5s + num_traces: 20000 + expected_new_traces_per_sec: 500 + policies: + - {name: errors, type: status_code, status_code: {status_codes: [ERROR]}} + - {name: slow, type: latency, latency: {threshold_ms: 500}} + - {name: baseline, type: probabilistic, probabilistic: {sampling_percentage: 10}} + +exporters: + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: {insecure: true} + sending_queue: {enabled: true, num_consumers: 2, queue_size: 500} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 30s} + otlp_http/loki: + endpoint: http://loki:3100/otlp + sending_queue: {enabled: true, num_consumers: 2, queue_size: 500} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 30s} + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: {enabled: true} + metric_expiration: 20s + +service: + telemetry: + metrics: + level: normal + readers: + - pull: + exporter: + prometheus: {host: 0.0.0.0, port: 8888} + logs: {level: warn} + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, tail_sampling, batch] + exporters: [otlp_grpc/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlp_http/loki] diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..bb3556c --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,75 @@ +# OpenTelemetry Collector (contrib). Один пайплайн на сигнал, у каждого — реальный backend: +# traces -> Tempo (OTLP/gRPC) +# metrics -> Prometheus (Prometheus-экспортёр: Prometheus сам забирает /metrics у Collector) +# logs -> Loki (OTLP/HTTP, нативный приём Loki) +# Collector — маршрутизатор/обработчик, не долговременное хранилище. Порты 4317/4318 доступны ТОЛЬКО во внутренней сети backend. +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + # Защита RAM Collector: при приближении к лимиту новые данные отклоняются (клиент SDK получает ошибку и отбрасывает + # пакет, приложение продолжает работать). limit_mib согласован с mem_limit контейнера. + memory_limiter: + check_interval: 1s + limit_mib: 280 + spike_limit_mib: 60 + batch: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 2s + # Гарантированное удаление чувствительных атрибутов, даже если приложение их случайно передаст. + attributes/scrub: + actions: + - {key: http.request.header.authorization, action: delete} + - {key: http.request.header.cookie, action: delete} + - {key: http.request.body, action: delete} + - {key: db.statement, action: delete} + - {key: url.query, action: delete} + - {key: http.target, action: delete} + # Tail-sampling (профиль «tail», см. collector-tail.yaml): здесь не используется. + +exporters: + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: {insecure: true} + # Ограниченная буферизация при недоступном backend: очередь фиксированного размера, повторы до 30 с, затем отбрасывание + # (потери считаются метрикой otelcol_exporter_send_failed_spans / otelcol_exporter_queue_size). + sending_queue: {enabled: true, num_consumers: 2, queue_size: 500} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 30s} + otlp_http/loki: + endpoint: http://loki:3100/otlp + sending_queue: {enabled: true, num_consumers: 2, queue_size: 500} + retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 30s} + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: {enabled: true} + metric_expiration: 20s + +service: + telemetry: + metrics: + level: normal + readers: + - pull: + exporter: + prometheus: {host: 0.0.0.0, port: 8888} + logs: + level: warn + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlp_grpc/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlp_http/loki] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..a030ea9 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,20 @@ +# Prometheus: хранилище метрик (retention 7 дней, volume promdata). Забирает метрики приложений через Collector +# (Prometheus-экспортёр :8889) и собственные метрики Collector (:8888): очереди экспортёров, отказы, drop из memory_limiter. +global: + scrape_interval: 5s + evaluation_interval: 5s + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: otel-collector-app-metrics + honor_labels: true + static_configs: + - targets: ["otel-collector:8889"] + - job_name: otel-collector-internal + static_configs: + - targets: ["otel-collector:8888"] + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] diff --git a/infra/prometheus/rules/messenger.yml b/infra/prometheus/rules/messenger.yml new file mode 100644 index 0000000..c9263df --- /dev/null +++ b/infra/prometheus/rules/messenger.yml @@ -0,0 +1,77 @@ +# Правила записи (SLI) и алерты. Runbook для каждого алерта — docs/runbooks.md (ссылка в аннотации). +# SLI доступности API: доля бизнес-запросов (без /health) без ответа 5xx. 4xx — ошибка клиента, в SLI не входит. +# SLO: 99.5 % за скользящее окно 30 дней => бюджет ошибок 0.5 %. Для демонстрации окна короткие (локальный стенд). +groups: + - name: messenger-sli + interval: 5s + rules: + - record: sli:http_requests:rate1m + expr: sum(rate(http_server_request_duration_seconds_count{http_route!~"/health.*"}[1m])) + - record: sli:http_errors:rate1m + expr: sum(rate(http_server_request_duration_seconds_count{http_route!~"/health.*",http_response_status_code=~"5.."}[1m])) + - record: sli:http_error_ratio:1m + expr: (sli:http_errors:rate1m / sli:http_requests:rate1m) or vector(0) + - record: sli:http_error_ratio:5m + expr: | + (sum(rate(http_server_request_duration_seconds_count{http_route!~"/health.*",http_response_status_code=~"5.."}[5m])) + / sum(rate(http_server_request_duration_seconds_count{http_route!~"/health.*"}[5m]))) or vector(0) + - record: sli:http_latency_p95:5m + expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{http_route!~"/health.*"}[5m]))) + - record: sli:http_latency_p95:1m + expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{http_route!~"/health.*"}[1m]))) + - record: sli:upload_error_ratio:1m + expr: | + (sum(rate(http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/attachments",http_response_status_code=~"5.."}[1m])) + / sum(rate(http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/attachments"}[1m]))) or vector(0) + - record: sli:upload_requests:rate1m + expr: sum(rate(http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/attachments"}[1m])) + - record: sli:time_to_ready_p95:5m + expr: histogram_quantile(0.95, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m]))) + - name: messenger-alerts + rules: + - alert: HighHttpErrorRate + expr: sli:http_error_ratio:1m > 0.05 and sli:http_requests:rate1m > 0.5 + for: 20s + labels: {severity: page} + annotations: + summary: "Доля 5xx бизнес-запросов API выше 5 %" + description: "Числитель — ответы 5xx, знаменатель — все запросы кроме /health. Значение: {{ $value | humanizePercentage }}." + runbook: "docs/runbooks.md#higherrorrate" + - alert: HighUploadErrorRate + expr: sli:upload_error_ratio:1m > 0.2 and sli:upload_requests:rate1m > 0.2 + for: 15s + labels: {severity: page} + annotations: + summary: "Более 20 % загрузок вложений завершаются 5xx" + description: "Маршрут загрузки критичен, но малообъёмен: общий SLI 5xx его деградацию не заметит, поэтому есть отдельный алерт. Значение: {{ $value | humanizePercentage }}." + runbook: "docs/runbooks.md#higherrorrate" + - alert: HighLatencyP95 + expr: sli:http_latency_p95:1m > 0.5 and sli:http_requests:rate1m > 0.5 + for: 20s + labels: {severity: ticket} + annotations: + summary: "p95 задержки бизнес-запросов выше 500 мс" + description: "p95 из суммарной гистограммы всех реплик: {{ $value | humanizeDuration }}. Различайте медленную БД (spans db.connection, db_client_connections_waiting) и медленный обработчик." + runbook: "docs/runbooks.md#highlatency" + - alert: QueueLagGrowing + expr: messenger_queue_oldest_job_age_seconds > 30 + for: 20s + labels: {severity: ticket} + annotations: + summary: "Самая старая задача в очереди ждёт дольше 30 с" + description: "Возраст старейшего задания: {{ $value }} с. Проверьте worker, брокер, S3." + runbook: "docs/runbooks.md#queuelag" + - alert: WorkerSilent + expr: absent(messenger_worker_capacity) + for: 30s + labels: {severity: ticket} + annotations: + summary: "Worker не присылает метрики" + runbook: "docs/runbooks.md#queuelag" + - alert: TelemetryExportFailing + expr: sum(rate(otelcol_exporter_send_failed_spans[1m])) + sum(rate(otelcol_exporter_send_failed_log_records[1m])) > 0 + for: 15s + labels: {severity: ticket} + annotations: + summary: "Collector не может отправить телеметрию в хранилище" + runbook: "docs/runbooks.md#telemetry" diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..3cdefa3 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,33 @@ +# Tempo: хранилище трасс (локальный диск, volume tempodata, retention 7 суток). Принимает OTLP/gRPC от Collector. +stream_over_http_enabled: true +server: + http_listen_port: 3200 + log_level: warn +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 +ingester: + max_block_duration: 1m +compactor: + compaction: + block_retention: 168h +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +# Лимиты приёма: значения по умолчанию (10 000 «живых» трасс на тенанта) отбрасывали до 30 % спанов под нагрузкой +# (метрика tempo_discarded_spans_total{reason="live_traces_exceeded"}) — найдено экспериментом «цена наблюдаемости». +overrides: + defaults: + ingestion: + max_traces_per_user: 150000 + rate_limit_bytes: 50000000 + burst_size_bytes: 100000000 +usage_report: + reporting_enabled: false diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 0000000..2effe2e --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,52 @@ +-- 0001: пользователи, чаты, участники, сообщения, сессии. +-- Источник истины для всех данных — PostgreSQL. Каждая миграция выполняется в транзакции. +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); +-- Уникальность username: гонка двух регистраций решается ограничением БД, а не проверкой в приложении. +CREATE UNIQUE INDEX users_username_key ON users (username); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +-- chat_created_at денормализован ради keyset-пагинации списка чатов пользователя без join+sort. +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + chat_created_at timestamptz NOT NULL, + ord smallint NOT NULL, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_chats ON chat_members (user_id, chat_created_at, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000) +); +-- История чата: WHERE chat_id = ? AND (created_at, id) > (?, ?) ORDER BY created_at, id LIMIT n +-- обслуживается этим индексом без сортировки и без сканирования всей истории. +CREATE INDEX messages_chat_history ON messages (chat_id, created_at, id); + +-- Серверные сессии. В БД хранится только SHA-256 токена: утечка БД не раскрывает рабочие токены. +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz +); +CREATE INDEX sessions_expires_at ON sessions (expires_at); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 0000000..a18c45f --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,17 @@ +-- 0002: общая идемпотентность отправки сообщений (lab3). +-- Область уникальности: пользователь + чат + ключ (метод и путь фиксированы: POST /chats/{id}/messages). +-- Запись ключа и сообщения — одна транзакция: побочный эффект и результат фиксируются атомарно. +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + idem_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, idem_key), + CONSTRAINT idem_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + -- сообщение вставляется после ключа в той же транзакции -> проверка внешнего ключа откладывается до COMMIT + CONSTRAINT idem_message_fk FOREIGN KEY (message_id) REFERENCES messages (id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED +); +CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at); diff --git a/migrations/0003_send_message_function.sql b/migrations/0003_send_message_function.sql new file mode 100644 index 0000000..f9e8dd8 --- /dev/null +++ b/migrations/0003_send_message_function.sql @@ -0,0 +1,35 @@ +-- 0003: отправка сообщения одной серверной функцией (оптимизация по итогам трека 1: меньше обращений к БД на запрос). +-- Семантика идентична версии «несколько запросов в транзакции» (migrations 0002 + код приложения): +-- проверка членства -> просроченный ключ удаляется -> INSERT ключа (ON CONFLICT DO NOTHING) -> INSERT сообщения. +-- Функция выполняется атомарно (один вызов = одна транзакция); гонку одинаковых ключей по-прежнему решает уникальный индекс. +CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision) +RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz, + request_hash bytea) +LANGUAGE plpgsql AS $$ +DECLARE + v_new uuid := gen_random_uuid(); + v_ins uuid; + v_created timestamptz; +BEGIN + IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN + RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz, NULL::bytea; + RETURN; + END IF; + DELETE FROM idempotency_keys k + WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key + AND k.created_at < now() - make_interval(secs => p_ttl); + INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id) + VALUES (p_user, p_chat, p_key, p_hash, v_new) + ON CONFLICT DO NOTHING + RETURNING message_id INTO v_ins; + IF v_ins IS NOT NULL THEN + INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text) + RETURNING created_at INTO v_created; + RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash; + ELSE + RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash + FROM idempotency_keys k JOIN messages m ON m.id = k.message_id + WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key; + END IF; +END; +$$; diff --git a/migrations/0004_attachments.sql b/migrations/0004_attachments.sql new file mode 100644 index 0000000..e1c15b9 --- /dev/null +++ b/migrations/0004_attachments.sql @@ -0,0 +1,90 @@ +-- 0004 (lab4): метаданные вложений в PostgreSQL, байты — в приватном S3-бакете. +-- Ключ объекта назначает сервер (att//original); filename — только отображаемое имя. +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + status text NOT NULL DEFAULT 'ready', + error_code text, + s3_key text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255) +); +CREATE UNIQUE INDEX attachments_s3_key_key ON attachments (s3_key); +CREATE INDEX attachments_chat ON attachments (chat_id); + +-- Связь сообщение <-> вложения (порядок сохраняется). Вложение может быть привязано к нескольким сообщениям чата. +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments (id), + position smallint NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); +CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id); + +-- Отправка сообщения с вложениями: заменяет функцию из 0003 (добавлен параметр p_atts и результат msg_atts). +DROP FUNCTION send_message(uuid, uuid, text, bytea, text, double precision); +CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision, + p_atts uuid[]) +RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz, + request_hash bytea, msg_atts uuid[]) +LANGUAGE plpgsql AS $$ +DECLARE + v_new uuid := gen_random_uuid(); + v_ins uuid; + v_created timestamptz; + v_found int; + v_failed int; +BEGIN + IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN + RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz, + NULL::bytea, NULL::uuid[]; + RETURN; + END IF; + IF cardinality(p_atts) > 0 THEN + SELECT count(*), count(*) FILTER (WHERE a.status = 'failed') INTO v_found, v_failed + FROM attachments a WHERE a.id = ANY (p_atts) AND a.chat_id = p_chat; + IF v_found <> cardinality(p_atts) THEN + RETURN QUERY SELECT 'attachment_not_found'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, + NULL::timestamptz, NULL::bytea, NULL::uuid[]; + RETURN; + END IF; + IF v_failed > 0 THEN + RETURN QUERY SELECT 'attachment_failed'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, + NULL::timestamptz, NULL::bytea, NULL::uuid[]; + RETURN; + END IF; + END IF; + DELETE FROM idempotency_keys k + WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key + AND k.created_at < now() - make_interval(secs => p_ttl); + INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id) + VALUES (p_user, p_chat, p_key, p_hash, v_new) + ON CONFLICT DO NOTHING + RETURNING message_id INTO v_ins; + IF v_ins IS NOT NULL THEN + INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text) + RETURNING created_at INTO v_created; + INSERT INTO message_attachments (message_id, attachment_id, position) + SELECT v_new, a, (o - 1)::smallint FROM unnest(p_atts) WITH ORDINALITY AS t(a, o); + RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash, p_atts; + ELSE + RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash, + COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) + FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[]) + FROM idempotency_keys k JOIN messages m ON m.id = k.message_id + WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key; + END IF; +END; +$$; diff --git a/migrations/0005_async_pipeline.sql b/migrations/0005_async_pipeline.sql new file mode 100644 index 0000000..51d3d48 --- /dev/null +++ b/migrations/0005_async_pipeline.sql @@ -0,0 +1,60 @@ +-- 0005 (lab5): асинхронная обработка вложений — состояния, варианты, outbox, журнал событий. +-- Расширение обратно совместимо с lab4: старые строки (ready) остаются валидными. +ALTER TABLE attachments + ADD COLUMN staging_key text, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN attempt integer NOT NULL DEFAULT 0, + ADD COLUMN job_id uuid, + ADD COLUMN lease_owner text, + ADD COLUMN lease_expires_at timestamptz, + ADD COLUMN processing_started_at timestamptz, + ADD COLUMN accepted_at timestamptz NOT NULL DEFAULT clock_timestamp(), + ADD COLUMN ready_at timestamptz; + +-- очередь/backlog и поиск зависших processing без полного скана +CREATE INDEX attachments_active ON attachments (status, accepted_at) WHERE status IN ('queued', 'processing'); +CREATE INDEX attachments_lease ON attachments (lease_expires_at) WHERE status = 'processing'; + +-- Результаты обработки. Ключ эффекта (attachment_id, pipeline_version, name): повторная запись не создаёт второй набор вариантов. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + s3_key text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')), + CONSTRAINT variants_content_type CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')) +); + +-- Transactional outbox: намерение «обработать файл» фиксируется В ТОЙ ЖЕ транзакции, что и строка вложения. +-- Relay публикует строки в брокер (publisher confirm) и только затем ставит published_at. available_at — отложенная +-- публикация (backoff между попытками). Доставка at-least-once; обработка идемпотентна. +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL DEFAULT 1, + attempt integer NOT NULL, + available_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX outbox_pending ON outbox (available_at) WHERE published_at IS NULL; + +-- Журнал событий пайплайна (аудит операторского replay, причины отказов; без секретов). +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + attachment_id uuid NOT NULL, + job_id uuid, + event text NOT NULL, + attempt integer, + detail text, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX job_events_attachment ON job_events (attachment_id, id); diff --git a/migrations/0006_outbox_trace_context.sql b/migrations/0006_outbox_trace_context.sql new file mode 100644 index 0000000..133e991 --- /dev/null +++ b/migrations/0006_outbox_trace_context.sql @@ -0,0 +1,3 @@ +-- 0006 (lab6): контекст трассировки в outbox. traceparent/tracestate запроса upload сохраняются вместе с заданием, +-- relay передаёт их в брокер (поля jobs.schema.json), worker продолжает ту же трассу: API -> outbox -> broker -> worker. +ALTER TABLE outbox ADD COLUMN traceparent text, ADD COLUMN tracestate text; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a74a3f2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,43 @@ +aio-pika==10.1.0 +aiobotocore==3.9.2 +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aioitertools==0.13.0 +aiormq==7.2.1 +aiosignal==1.4.0 +argon2-cffi-bindings==26.1.0 +argon2-cffi==23.1.0 +asyncpg==0.30.0 +attrs==26.1.0 +botocore==1.43.106 +cffi==2.1.1 +click==8.5.0 +frozenlist==1.8.0 +googleapis-common-protos==1.75.5 +h11==0.16.0 +httptools==0.8.0 +idna==3.20 +jmespath==1.1.0 +multidict==6.9.1 +opentelemetry-api==1.45.0 +opentelemetry-exporter-http-transport==0.66b0 +opentelemetry-exporter-otlp-common==0.66b0 +opentelemetry-exporter-otlp-proto-common==1.45.0 +opentelemetry-exporter-otlp-proto-http==1.45.0 +opentelemetry-proto==1.45.0 +opentelemetry-sdk==1.45.0 +opentelemetry-semantic-conventions==0.66b0 +pamqp==4.0.1 +pillow==12.3.0 +propcache==0.5.4 +protobuf==7.36.2 +pycparser==3.0 +python-dateutil==2.9.0.post0 +redis==5.2.1 +six==1.17.0 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.0 +uvloop==0.21.0; sys_platform != "win32" +wrapt==2.5.0 +yarl==1.25.1 diff --git a/research/failover_sequence.py b/research/failover_sequence.py new file mode 100644 index 0000000..dbf56a1 --- /dev/null +++ b/research/failover_sequence.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Уровень 4: последовательная остановка каждой API-реплики под нагрузкой (с восстановлением предыдущей). + +Сценарий (стенд `docker compose up -d` с 2 репликами; нагрузка идёт через LB непрерывно ~70 с): + t=10 остановить реплику A (docker stop --time 0 = мгновенный отказ процесса) + t=25 восстановить (scripts/restore-replicas.sh) + t=40 остановить реплику B + t=55 восстановить +Для каждого отказа считаем: RPS до/во время/после, число ошибочных попыток, время до прекращения ошибок +(= обнаружение отказа балансировщиком), сохранность подтверждённых сообщений и отсутствие дублей. + + python research/failover_sequence.py --out evidence +""" +import argparse +import asyncio +import json +import os +import subprocess +import sys +import threading +import time + +sys.path.insert(0, os.path.dirname(__file__)) +import loadgen # noqa: E402 + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +BASE = "http://127.0.0.1:8080" + + +def sh(*args): + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True).stdout.strip() + + +def api_ids(): + return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=mtusi-lab6", + "--filter", "label=com.docker.compose.service=api").split() + + +def scenario(rec, plan): + """Выполняется в отдельном потоке: действия по расписанию относительно rec.t0.""" + for at, action in plan: + time.sleep(max(0, rec.t0 + at - time.time())) + t = time.time() - rec.t0 + if action == "stop": + target = api_ids()[0] + subprocess.run(["docker", "stop", "--time", "0", target], capture_output=True) + rec.marks.append({"t": round(t, 2), "action": "stop", "container": target}) + else: + subprocess.run(["sh", "scripts/restore-replicas.sh"], cwd=ROOT, capture_output=True) + rec.marks.append({"t": round(t, 2), "action": "restore"}) + + +async def main(args): + stand = loadgen.Stand(BASE) + await stand.setup() + rec = loadgen.Recorder() + rec.marks = [] + plan = [(10, "stop"), (25, "restore"), (40, "stop"), (55, "restore")] + threading.Thread(target=scenario, args=(rec, plan), daemon=True).start() + await loadgen.run_load(stand, rec, args.seconds, args.concurrency) + await asyncio.sleep(2) + check = await loadgen.verify(stand, rec) + stops = [m for m in rec.marks if m["action"] == "stop"] + restores = [m for m in rec.marks if m["action"] == "restore"] + windows = [] + for i, stop in enumerate(stops): + t0 = stop["t"] + errs = [e for e in rec.events if e[1] == "POST" and e[2] != 201 and t0 - 0.2 <= e[0] <= t0 + 10] + last_err = max((e[0] for e in errs), default=t0) + restore_t = restores[i]["t"] if i < len(restores) else t0 + 15 + windows.append({"stopped_container": stop["container"], "stop_at_s": t0, + "rps_before": loadgen.summarize(rec, t0 - 8, t0 - 0.5).get("ok_rps"), + "rps_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("ok_rps"), + "rps_after_restore": loadgen.summarize(rec, restore_t + 8, restore_t + 14).get("ok_rps"), + "failed_attempts_within_10s": len(errs), + "error_statuses": sorted({str(e[2]) for e in errs}), + "seconds_until_errors_stop": round(last_err - t0, 2) if errs else 0.0, + "p95_ms_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("p95_ms")}) + series = loadgen.per_second(rec) + report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "marks": rec.marks, "windows": windows, + "total": loadgen.summarize(rec), "verification": check, + "invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0, + "timeline": [{"t": t, "ok": b["ok"], "err": b["err"], "p95_ms": loadgen.pct(b["lat"], .95), "instances": len(b["instances"])} + for t, b in sorted(series.items())]} + with open(os.path.join(args.out, "failover_sequence.json"), "w", encoding="utf-8") as fh: + json.dump(report, fh, ensure_ascii=False, indent=2) + plot(report, os.path.join(args.out, "failover_sequence.png")) + print(json.dumps({k: v for k, v in report.items() if k != "timeline"}, ensure_ascii=False, indent=2)) + + +def plot(report, path): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + tl = report["timeline"] + t = [x["t"] for x in tl] + fig, ax = plt.subplots(3, 1, figsize=(11, 8.5), sharex=True) + ax[0].plot(t, [x["ok"] for x in tl], color="#2e7d32"); ax[0].set_ylabel("успешных POST/с"); ax[0].set_title("Пропускная способность") + ax[1].bar(t, [x["err"] for x in tl], color="#c62828"); ax[1].set_ylabel("ошибочных попыток/с"); ax[1].set_title("Ошибки (до повтора с тем же ключом)") + ax[2].plot(t, [x["p95_ms"] or 0 for x in tl], color="#1565c0"); ax[2].set_ylabel("p95, мс"); ax[2].set_xlabel("время, с"); ax[2].set_title("Латентность успешных POST") + for a in ax: + for m in report["marks"]: + a.axvline(m["t"], color="red" if m["action"] == "stop" else "green", ls="--", lw=1) + ax[0].text(0.01, 0.05, "красная линия — остановка реплики, зелёная — восстановление", transform=ax[0].transAxes, fontsize=9) + fig.tight_layout(); fig.savefig(path, dpi=110) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="evidence") + ap.add_argument("--seconds", type=int, default=70) + ap.add_argument("--concurrency", type=int, default=16) + asyncio.run(main(ap.parse_args())) diff --git a/research/incidents.py b/research/incidents.py new file mode 100644 index 0000000..97b0b4a --- /dev/null +++ b/research/incidents.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Уровень 4: контролируемые инциденты и проверка алертов firing -> resolved. + +Фоном идёт нагрузка (сообщения + загрузки). Инциденты и ожидаемые алерты: + 1. worker_down — остановка worker: растёт возраст старейшей задачи -> QueueLagGrowing; после запуска — resolved. + 2. slow_db — триггер в БД добавляет pg_sleep(0.8) на вставку сообщения: растёт p95 -> HighLatencyP95; + причина отличается от медленного обработчика по spans db.connection и метрике ожидания пула. + 3. s3_down — остановка S3: загрузки дают 503 -> HighUploadErrorRate (маршрут критичен, но малообъёмен). + 4. telemetry_backend — остановка Tempo и Loki: API продолжает отвечать, Collector ограничивает очередь/RAM -> TelemetryExportFailing. +Для каждого фиксируем времена перехода алертов (по Prometheus API), ключевые метрики и «улики» из Tempo/Loki. + python research/incidents.py --out evidence +""" +import argparse +import asyncio +import base64 +import json +import os +import subprocess +import sys +import threading +import time +import urllib.parse +import uuid + +import aiohttp + +sys.path.insert(0, os.path.dirname(__file__)) +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +BASE = "http://127.0.0.1:8080" + + +def sh(*args, stdin=None): + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", input=stdin).stdout.strip() + + +def dc(*args): + return sh("docker", "compose", "--env-file", ".env", *args) + + +def prom_get(path): + return json.loads(dc("exec", "-T", "prometheus", "wget", "-q", "-O", "-", "http://127.0.0.1:9090" + path) or "{}") + + +def prom_value(query): + r = prom_get("/api/v1/query?query=" + urllib.parse.quote(query)) + res = r.get("data", {}).get("result", []) + return float(res[0]["value"][1]) if res else None + + +def alerts(): + r = prom_get("/api/v1/alerts") + return {a["labels"]["alertname"]: a["state"] for a in r.get("data", {}).get("alerts", [])} + + +def net_get(url, accept=None): + cmd = ["exec", "-T", "prometheus", "wget", "-q", "-O", "-"] + (["--header", "Accept: " + accept] if accept else []) + [url] + return dc(*cmd) + + +class Recorder(threading.Thread): + def __init__(self): + super().__init__(daemon=True) + self.t0, self.rows, self.stop = time.time(), [], threading.Event() + + def run(self): + while not self.stop.is_set(): + t = time.time() - self.t0 + row = {"t": round(t, 1), "alerts": alerts(), + "queue_oldest_age_s": prom_value("max(messenger_queue_oldest_job_age_seconds)"), + "error_ratio_1m": prom_value("sli:http_error_ratio:1m"), + "p95_s": prom_value("sli:http_latency_p95:1m"), + "db_waiting": prom_value("sum(db_client_connections_waiting)"), + "db_used": prom_value('sum(db_client_connections_usage{state="used"})'), + "rps": prom_value("sli:http_requests:rate1m")} + self.rows.append(row) + self.stop.wait(3) + + +async def register(s): + name = "inc_%s" % uuid.uuid4().hex[:12] + pw = "incident-test-pw-" + uuid.uuid4().hex[:6] + async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "i", "password": pw}) as r: + user = await r.json() + cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode() + async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r: + token = (await r.json())["token"] + async with s.post(BASE + "/api/v1/chats", json={"title": "inc", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r: + chat = await r.json() + return token, chat["id"] + + +async def traffic(stop, png, counters): + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=15), connector=aiohttp.TCPConnector(limit=40)) as s: + token, chat = await register(s) + hdr = {"Authorization": "Bearer " + token} + + async def msgs(): + while not stop.is_set(): + try: + async with s.post("%s/api/v1/chats/%s/messages" % (BASE, chat), json={"text": "inc-" + uuid.uuid4().hex[:8]}, + headers=dict(hdr, **{"Idempotency-Key": uuid.uuid4().hex})) as r: + await r.read() + counters["msg_%d" % (r.status // 100)] = counters.get("msg_%d" % (r.status // 100), 0) + 1 + except Exception: + counters["msg_err"] = counters.get("msg_err", 0) + 1 + await asyncio.sleep(0.1) + + async def uploads(): + while not stop.is_set(): + try: + async with s.post("%s/api/v1/chats/%s/attachments?filename=i.png" % (BASE, chat), data=png, + headers=dict(hdr, **{"Content-Type": "image/png"})) as r: + await r.read() + counters["up_%d" % (r.status // 100)] = counters.get("up_%d" % (r.status // 100), 0) + 1 + except Exception: + counters["up_err"] = counters.get("up_err", 0) + 1 + await asyncio.sleep(0.3) + await asyncio.gather(*([msgs() for _ in range(4)] + [uploads() for _ in range(10)])) + + +def tempo_search(traceql, limit=20): + now = int(time.time()) + url = "http://tempo:3200/api/search?q=%s&limit=%d&start=%d&end=%d" % (urllib.parse.quote(traceql), limit, now - 900, now + 5) + raw = net_get(url) + try: + return json.loads(raw).get("traces", []) + except ValueError: + return [] + + +def loki_count(query): + now = int(time.time() * 1e9) + url = "http://loki:3100/loki/api/v1/query_range?limit=500&start=%d&end=%d&query=%s" % (now - 900 * 10 ** 9, now, urllib.parse.quote(query)) + try: + data = json.loads(net_get(url)) + return sum(len(s["values"]) for s in data["data"]["result"]) + except ValueError: + return None + + +def wait_until(pred, seconds): + deadline = time.time() + seconds + while time.time() < deadline: + if pred(): + return True + time.sleep(2) + return False + + +def run_incident(name, expected_alert, start, stop, hold, evidence, rec): + t_start = time.time() - rec.t0 + start() + fired = wait_until(lambda: alerts().get(expected_alert) == "firing", hold) + t_fired = time.time() - rec.t0 if fired else None + ev = evidence() if evidence else {} + ev["alerts_in_state_firing"] = sorted(k for k, v in alerts().items() if v == "firing") + stop() + t_recovery = time.time() - rec.t0 + resolved = wait_until(lambda: expected_alert not in alerts(), 120) + t_resolved = time.time() - rec.t0 if resolved else None + result = {"incident": name, "alert": expected_alert, "injected_at_s": round(t_start, 1), + "firing_at_s": round(t_fired, 1) if t_fired else None, "detection_seconds": round(t_fired - t_start, 1) if t_fired else None, + "recovered_cmd_at_s": round(t_recovery, 1), "resolved_at_s": round(t_resolved, 1) if t_resolved else None, + "fired": fired, "resolved": resolved, "evidence": ev} + if t_fired and t_resolved: + result["time_to_resolve_after_recovery_s"] = round(t_resolved - t_recovery, 1) + time.sleep(6) + return result + + +def orchestrate(rec, results, args): + time.sleep(25) # прогрев: метрики и алерты стабилизируются + # 1. worker down + results.append(run_incident( + "worker_down", "QueueLagGrowing", lambda: dc("stop", "worker"), lambda: dc("start", "worker"), 150, + lambda: {"queue_oldest_age_s": prom_value("max(messenger_queue_oldest_job_age_seconds)"), "worker_capacity_series_present": prom_value("sum(messenger_worker_capacity)") is not None, + "jobs_ready_rate_1m": prom_value('sum(rate(messenger_jobs_processed_total{outcome="ready"}[1m]))'), + "diagnosis": "возраст старейшей задачи растёт при нулевой скорости обработки jobs (worker не отдаёт метрики) -> остановлен worker, а не медленная БД"}, + rec)) + # 2. slow DB + trigger_sql = ("CREATE OR REPLACE FUNCTION slow_insert() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_sleep(0.8); RETURN NEW; END $$;" + "CREATE TRIGGER slow_messages BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION slow_insert();") + pg = ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-c"] + results.append(run_incident( + "slow_db", "HighLatencyP95", lambda: dc(*pg, trigger_sql), lambda: dc(*pg, "DROP TRIGGER IF EXISTS slow_messages ON messages; DROP FUNCTION IF EXISTS slow_insert();"), 120, + lambda: {"p95_s": prom_value("sli:http_latency_p95:1m"), "db_pool_waiting": prom_value("sum(db_client_connections_waiting)"), + "slow_traces_with_long_db_span": len(tempo_search('{ resource.service.name = "messenger-api" && name = "db.connection" && duration > 300ms }')), + "slow_http_post_messages_traces": len(tempo_search('{ name =~ "HTTP POST.*messages" && duration > 300ms }')), + "diagnosis": "p95 высокий, а время в спанах db.connection сопоставимо с длительностью запроса и растёт ожидание пула -> медленная БД, а не обработчик"}, + rec)) + # 3. S3 down + results.append(run_incident( + "s3_down", "HighUploadErrorRate", lambda: dc("stop", "seaweedfs"), lambda: dc("start", "seaweedfs"), 120, + lambda: {"upload_error_ratio_1m": prom_value("sli:upload_error_ratio:1m"), "global_error_ratio_1m": prom_value("sli:http_error_ratio:1m"), "error_traces": len(tempo_search('{ status = error && resource.service.name = "messenger-api" }')), + "s3_put_failed_log_lines": loki_count('{service_name="messenger-api"} |= "s3_put_failed"'), + "diagnosis": "рост доли 5xx на маршруте загрузки при нормальных остальных; спаны s3.put_object с ошибкой и логи s3_put_failed -> недоступен S3"}, + rec)) + # 4. telemetry backend down + results.append(run_incident( + "telemetry_backend", "TelemetryExportFailing", lambda: dc("stop", "tempo", "loki"), lambda: dc("start", "tempo", "loki"), 120, + lambda: {"api_error_ratio_1m": prom_value("sli:http_error_ratio:1m"), "api_rps": prom_value("sli:http_requests:rate1m"), + "collector_queue_size": prom_value("max(otelcol_exporter_queue_size)"), "collector_send_failed_spans_rate": prom_value("sum(rate(otelcol_exporter_send_failed_spans[1m]))"), + "collector_rss_mib": round((prom_value("max(otelcol_process_memory_rss)") or 0) / 1048576, 1), + "diagnosis": "ошибки экспорта растут, очередь ограничена; бизнес-метрики API без роста ошибок -> сбой хранилища телеметрии, не приложения"}, + rec)) + + +async def main(args): + from PIL import Image + import io + buf = io.BytesIO() + Image.new("RGB", (320, 200), (30, 90, 160)).save(buf, "PNG") + stop_flag, counters, results = asyncio.Event(), {}, [] + rec = Recorder() + rec.start() + t = threading.Thread(target=orchestrate, args=(rec, results, args), daemon=True) + t.start() + traffic_task = asyncio.create_task(traffic(stop_flag, buf.getvalue(), counters)) + while t.is_alive(): + await asyncio.sleep(1) + stop_flag.set() + await asyncio.sleep(1) + traffic_task.cancel() + rec.stop.set() + collector_rss = None + report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "traffic_counters": counters, "incidents": results, "timeline": rec.rows} + with open(os.path.join(args.out, "incidents.json"), "w", encoding="utf-8") as fh: + json.dump(report, fh, ensure_ascii=False, indent=2) + plot(report, os.path.join(args.out, "incidents.png")) + print(json.dumps({"incidents": results, "traffic": counters}, ensure_ascii=False, indent=2)) + + +def plot(report, path): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + rows = report["timeline"] + t = [r["t"] for r in rows] + fig, ax = plt.subplots(4, 1, figsize=(12, 11), sharex=True) + ax[0].plot(t, [r["queue_oldest_age_s"] or 0 for r in rows], color="#6a1b9a"); ax[0].axhline(30, color="red", ls="--"); ax[0].set_ylabel("с"); ax[0].set_title("Возраст старейшей задачи (порог алерта 30 с)") + ax[1].plot(t, [r["p95_s"] or 0 for r in rows], color="#1565c0"); ax[1].axhline(0.5, color="red", ls="--"); ax[1].set_ylabel("с"); ax[1].set_title("p95 задержки API (порог 0.5 с)") + ax[2].plot(t, [(r["error_ratio_1m"] or 0) * 100 for r in rows], color="#c62828"); ax[2].axhline(5, color="red", ls="--"); ax[2].set_ylabel("%"); ax[2].set_title("Доля 5xx (порог 5 %)") + names = ["QueueLagGrowing", "HighLatencyP95", "HighHttpErrorRate", "TelemetryExportFailing"] + for i, n in enumerate(names): + ax[3].fill_between(t, i, [i + 0.8 if r["alerts"].get(n) == "firing" else i for r in rows], step="post", alpha=0.7, label=n) + ax[3].set_yticks([i + 0.4 for i in range(4)]); ax[3].set_yticklabels(names); ax[3].set_title("Алерты в состоянии firing"); ax[3].set_xlabel("время, с") + for inc in report["incidents"]: + for a in ax: + a.axvline(inc["injected_at_s"], color="gray", ls=":") + if inc["resolved_at_s"]: + a.axvline(inc["resolved_at_s"], color="green", ls=":") + fig.tight_layout(); fig.savefig(path, dpi=105) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="evidence") + asyncio.run(main(ap.parse_args())) diff --git a/research/loadgen.py b/research/loadgen.py new file mode 100644 index 0000000..3da976e --- /dev/null +++ b/research/loadgen.py @@ -0,0 +1,172 @@ +"""Генератор нагрузки для экспериментов lab6+ (asyncio + aiohttp, closed-loop). + +Каждый воркер выполняет цикл: POST сообщения с новым Idempotency-Key (90 %) или GET первой страницы истории (10 %). +При ошибке соединения/5xx POST повторяется с ТЕМ ЖЕ ключом (до RETRIES раз) — именно так клиент безопасно +повторяет неподтверждённую запись. Результат: по секундам успехи/ошибки/латентности, список подтверждённых +сообщений (для сверки с историей), число повторов. +""" +import asyncio +import base64 +import random +import time +import uuid + +import aiohttp + +RETRIES = 3 + + +class Stand: + """Набор пользователей/чатов, созданных через API (через LB), для одного эксперимента.""" + + def __init__(self, base, users=20, chats=20): + self.base, self.n_users, self.n_chats = base, users, chats + self.users, self.chats = [], [] + + async def setup(self): + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as s: + for i in range(self.n_users): + name = "ld_%s" % uuid.uuid4().hex[:14] + pw = "load-test-password-" + uuid.uuid4().hex[:8] + async with s.post(self.base + "/api/v1/users", json={"username": name, "display_name": "load %d" % i, "password": pw}) as r: + user = await r.json() + assert r.status == 201, user + cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode() + async with s.post(self.base + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r: + sess = await r.json() + assert r.status == 201, sess + self.users.append({"id": user["id"], "token": sess["token"]}) + for i in range(self.n_chats): + members = random.sample(self.users, 3) + owner = members[0] + async with s.post(self.base + "/api/v1/chats", json={"title": "load %d" % i, "member_ids": [m["id"] for m in members]}, + headers={"Authorization": "Bearer " + owner["token"]}) as r: + chat = await r.json() + assert r.status == 201, chat + self.chats.append({"id": chat["id"], "members": members}) + + +class Recorder: + def __init__(self): + self.t0 = time.time() + self.events = [] # (t, kind, status, latency_ms, instance) + self.confirmed = [] # (chat_id, message_id, text) + self.retries = 0 + self.final_failures = 0 + self.unconfirmed = [] # (chat_id, key, text) — финально не подтверждены (могли записаться!) + + +async def worker(stand, rec, deadline_fn, read_ratio=0.1, text_size=0, think=0.0): + connector = aiohttp.TCPConnector(limit=2) + timeout = aiohttp.ClientTimeout(total=5) + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as s: + n = 0 + while not deadline_fn(): + chat = random.choice(stand.chats) + user = random.choice(chat["members"]) + hdr = {"Authorization": "Bearer " + user["token"]} + if random.random() < read_ratio: + t = time.perf_counter() + try: + async with s.get("%s/api/v1/chats/%s/messages?limit=20" % (stand.base, chat["id"]), headers=hdr) as r: + await r.read() + rec.events.append((time.time() - rec.t0, "GET", r.status, (time.perf_counter() - t) * 1000, r.headers.get("X-Instance-Id"))) + except Exception as exc: + rec.events.append((time.time() - rec.t0, "GET", type(exc).__name__, (time.perf_counter() - t) * 1000, None)) + continue + key = uuid.uuid4().hex + text = "m-%s-%d" % (key[:8], n) + "я" * text_size + n += 1 + for attempt in range(RETRIES + 1): + t = time.perf_counter() + try: + async with s.post("%s/api/v1/chats/%s/messages" % (stand.base, chat["id"]), json={"text": text}, + headers=dict(hdr, **{"Idempotency-Key": key})) as r: + body = await r.json(content_type=None) + status, inst = r.status, r.headers.get("X-Instance-Id") + except Exception as exc: + status, inst, body = type(exc).__name__, None, None + rec.events.append((time.time() - rec.t0, "POST", status, (time.perf_counter() - t) * 1000, inst)) + if status == 201: + rec.confirmed.append((chat["id"], body["id"], text)) + break + if attempt < RETRIES and (not isinstance(status, int) or status >= 500 or status == 429): + rec.retries += 1 + await asyncio.sleep(0.05 * (attempt + 1)) + continue + rec.final_failures += 1 + rec.unconfirmed.append((chat["id"], key, text)) + break + if think: + await asyncio.sleep(think) + + +async def run_load(stand, rec, seconds, concurrency, **kw): + end = time.time() + seconds + await asyncio.gather(*(worker(stand, rec, lambda: time.time() > end, **kw) for _ in range(concurrency))) + + +async def verify(stand, rec): + """Сверка: каждое подтверждённое сообщение читается, дублей по тексту нет, неподтверждённые встречаются не более 1 раза.""" + timeout = aiohttp.ClientTimeout(total=30) + stored = {} + async with aiohttp.ClientSession(timeout=timeout) as s: + for chat in stand.chats: + hdr = {"Authorization": "Bearer " + chat["members"][0]["token"]} + cursor, items = None, [] + while True: + url = "%s/api/v1/chats/%s/messages?limit=100%s" % (stand.base, chat["id"], "&cursor=" + cursor if cursor else "") + async with s.get(url, headers=hdr) as r: + page = await r.json() + assert r.status == 200, page + items += page["items"] + cursor = page["next_cursor"] + if not cursor: + break + stored[chat["id"]] = items + ids = {m["id"] for items in stored.values() for m in items} + texts = [m["text"] for items in stored.values() for m in items] + confirmed_missing = [c for c in rec.confirmed if c[1] not in ids] + dup_texts = len(texts) - len(set(texts)) + unconfirmed_present = sum(1 for c in rec.unconfirmed if c[2] in set(texts)) + return {"stored_messages": len(ids), "confirmed_201": len(rec.confirmed), "confirmed_missing_in_history": len(confirmed_missing), + "duplicate_texts_in_history": dup_texts, "unconfirmed_requests": len(rec.unconfirmed), + "unconfirmed_that_were_written": unconfirmed_present, "retries": rec.retries, "final_failures": rec.final_failures} + + +def per_second(rec, kind="POST"): + buckets = {} + for t, k, status, lat, inst in rec.events: + if k != kind: + continue + b = buckets.setdefault(int(t), {"ok": 0, "err": 0, "lat": [], "instances": set()}) + if status == 201 or (kind == "GET" and status == 200): + b["ok"] += 1 + b["lat"].append(lat) + if inst: + b["instances"].add(inst) + else: + b["err"] += 1 + return buckets + + +def pct(values, p): + if not values: + return None + values = sorted(values) + return round(values[max(0, int(len(values) * p + 0.9999) - 1)], 2) + + +def summarize(rec, t_from=None, t_to=None, kind="POST"): + rows = [e for e in rec.events if e[1] == kind and (t_from is None or e[0] >= t_from) and (t_to is None or e[0] < t_to)] + if not rows: + return {} + dur = (t_to if t_to is not None else max(e[0] for e in rows)) - (t_from if t_from is not None else min(e[0] for e in rows)) + ok = [e[3] for e in rows if e[2] == 201 or (kind == "GET" and e[2] == 200)] + errs = {} + for e in rows: + if not (e[2] == 201 or (kind == "GET" and e[2] == 200)): + errs[str(e[2])] = errs.get(str(e[2]), 0) + 1 + return {"requests": len(rows), "ok": len(ok), "errors": errs, "seconds": round(dur, 2), "ok_rps": round(len(ok) / dur, 1) if dur > 0 else None, + "error_ratio": round((len(rows) - len(ok)) / len(rows), 4), "p50_ms": pct(ok, .5), "p95_ms": pct(ok, .95), "p99_ms": pct(ok, .99)} diff --git a/research/obs_cost.py b/research/obs_cost.py new file mode 100644 index 0000000..4d76eb7 --- /dev/null +++ b/research/obs_cost.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Трек 1 (цена наблюдаемости), lab6. + +Профили при ОДИНАКОВОЙ нагрузке (8 процессов-клиентов, 64 одновременных, 90 % POST сообщений / 10 % GET, 30 с после прогрева): + off — OTEL_ENABLED=0 (SDK не создаётся, экспорта нет) + on100 — телеметрия включена, head-sampling 100 % трасс + head10 — head-sampling 10 % (ParentBased + TraceIdRatio в SDK) + tail — SDK отправляет 100 %, Collector оставляет errors + медленные (>500 мс) + 10 % остальных (tail_sampling) +Во время замера внедряются отказы: 2 % запросов дают 500 и 2 % задерживаются на 0.6 с (одинаково для всех профилей) — это «редкие +ошибки и медленные запросы», судьбу которых в Tempo мы сравниваем. +Метрики: RPS, p50/p95/p99, CPU/RSS API и Collector (docker stats), объём телеметрии (принятые Collector спаны/логи/точки метрик), +потери (отказы приёма Collector), размер хранилищ Tempo/Loki, доля сохранённых ошибочных и медленных трасс. + python research/obs_cost.py --out evidence +""" +import argparse +import asyncio +import json +import multiprocessing as mp +import os +import statistics +import subprocess +import sys +import threading +import time +import urllib.parse + +sys.path.insert(0, os.path.dirname(__file__)) +import loadgen # noqa: E402 + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +BASE = "http://127.0.0.1:8080" +PROFILES = { + "off": {"OTEL_ENABLED": "0", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml"}, + "on100": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml"}, + "head10": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml"}, + "tail": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector-tail.yaml"}, + # то же, что on100, но большая очередь/пакет экспорта SDK: проверка гипотезы «часть спанов теряется очередью SDK» + "on100_bigq": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml", + "OTEL_QUEUE_SIZE": "65536", "OTEL_BATCH_SIZE": "2048"}, +} + + +def sh(*args, env=None): + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", env=dict(os.environ, **(env or {}))).stdout.strip() + + +def dc(*args, env=None): + return sh("docker", "compose", "--env-file", ".env", *args, env=env) + + +def net_get(url): + return dc("exec", "-T", "prometheus", "wget", "-q", "-O", "-", url) + + +def prom_value(query): + try: + r = json.loads(net_get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(query)))["data"]["result"] + return float(r[0]["value"][1]) if r else None + except (ValueError, KeyError): + return None + + +def apply_profile(profile, faults=True): + env = dict(PROFILES[profile], API_FAULT_INJECTION="http500:0.02,slow:0.02:0.6" if faults else "") + dc("up", "-d", "--no-deps", "--force-recreate", "otel-collector", env=env) + dc("up", "-d", "--no-deps", "--force-recreate", "api", "worker", env=env) + deadline = time.time() + 90 + while time.time() < deadline: + ids = dc("ps", "-q", "api").split() + if len(ids) == 2 and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids): + break + time.sleep(1) + time.sleep(8) + + +def child(args): + stand_data, seconds, concurrency, seed, t0 = args + import random + random.seed(seed) + stand = loadgen.Stand(BASE) + stand.users, stand.chats = stand_data["users"], stand_data["chats"] + rec = loadgen.Recorder() + rec.t0 = t0 + + async def go(): + await loadgen.run_load(stand, rec, seconds, concurrency) + asyncio.run(go()) + return rec.events + + +class Sampler(threading.Thread): + def __init__(self): + super().__init__(daemon=True) + self.rows, self.stop = [], threading.Event() + + def run(self): + while not self.stop.is_set(): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.CPUPerc}} {{.MemUsage}}"], capture_output=True, text=True).stdout.splitlines() + self.rows.append((time.time(), [l.split() for l in out if "mtusi-lab6" in l])) + + def summary(self, t0, t1): + agg = {} + for ts, rows in self.rows: + if not t0 <= ts <= t1: + continue + for r in rows: + svc = r[0].replace("mtusi-lab6-", "").rsplit("-", 1)[0] + mem = float(r[2].replace("MiB", "").replace("GiB", "")) * (1024 if "GiB" in r[2] else 1) + agg.setdefault(svc, {"cpu": [], "mem": []}) + agg[svc]["cpu"].append(float(r[1].rstrip("%"))) + agg[svc]["mem"].append(mem) + return {k: {"avg_cpu_percent": round(statistics.mean(v["cpu"]), 1), "peak_mem_mib": round(max(v["mem"]), 1)} for k, v in agg.items()} + + +def volume_bytes(volume): + out = sh("docker", "run", "--rm", "-v", "%s:/v:ro" % volume, "alpine:3.20", "du", "-sk", "/v") + try: + return int(out.split()[0]) * 1024 + except (ValueError, IndexError): + return None + + +def counter(name): + return prom_value("sum(%s)" % name) or 0.0 + + +def tempo_count(traceql, since, limit=5000): + url = "http://tempo:3200/api/search?q=%s&limit=%d&start=%d&end=%d" % (urllib.parse.quote(traceql), limit, int(since), int(time.time()) + 5) + try: + return len(json.loads(net_get(url)).get("traces", [])) + except ValueError: + return None + + +def run_profile(profile, args, stand_data): + apply_profile(profile) + # прогрев + t0 = time.time() + with mp.get_context("spawn").Pool(args.procs) as pool: + pool.map(child, [(stand_data, 5, args.concurrency // args.procs, i, t0) for i in range(args.procs)]) + time.sleep(3) + before = {k: counter(k) for k in ("otelcol_receiver_accepted_spans", "otelcol_receiver_accepted_log_records", + "otelcol_receiver_accepted_metric_points", "otelcol_receiver_refused_spans", + "otelcol_exporter_sent_spans", "otelcol_exporter_send_failed_spans")} + vol_before = {"tempo": volume_bytes("mtusi-lab6_tempodata"), "loki": volume_bytes("mtusi-lab6_lokidata")} + sampler = Sampler() + sampler.start() + t_start = time.time() + with mp.get_context("spawn").Pool(args.procs) as pool: + results = pool.map(child, [(stand_data, args.seconds, args.concurrency // args.procs, 100 + i, t_start) for i in range(args.procs)]) + t_end = time.time() + sampler.stop.set() + rec = loadgen.Recorder() + rec.t0 = t_start + for events in results: + rec.events.extend(events) + post = loadgen.summarize(rec, 0, args.seconds, "POST") + time.sleep(args.flush) # дать SDK/Collector/хранилищам дописать данные + after = {k: counter(k) for k in before} + vol_after = {"tempo": volume_bytes("mtusi-lab6_tempodata"), "loki": volume_bytes("mtusi-lab6_lokidata")} + # «внедрённые» отказы считаем на стороне клиента (счётчики Prometheus нестабильны при пересоздании реплик): + # ответы 500 и ответы медленнее 500 мс (внедряется задержка 0.6 с; фоновые запросы быстрее) + posts = [e for e in rec.events if e[1] == "POST"] + injected_err = sum(1 for e in posts if e[2] == 500) + injected_slow = sum(1 for e in posts if e[2] == 201 and e[3] > 500) + kept_err = tempo_count('{ status = error && resource.service.name = "messenger-api" && name =~ "HTTP.*" }', t_start - 5) + kept_slow = tempo_count('{ resource.service.name = "messenger-api" && name =~ "HTTP.*" && duration > 500ms }', t_start - 5) + kept_all = tempo_count('{ resource.service.name = "messenger-api" && name =~ "HTTP.*" }', t_start - 5) + out = {"profile": profile, "post": post, "docker_stats": sampler.summary(t_start + 2, t_end), + "collector_delta": {k.replace("otelcol_", ""): round(after[k] - before[k]) for k in before}, + "tempo_bytes_delta": (vol_after["tempo"] - vol_before["tempo"]) if vol_after["tempo"] and vol_before["tempo"] else None, + "loki_bytes_delta": (vol_after["loki"] - vol_before["loki"]) if vol_after["loki"] and vol_before["loki"] else None, + "errors_injected_500": injected_err, "slow_requests_gt500ms": injected_slow, + "traces_in_tempo": {"all_http": kept_all, "error": kept_err, "slow_gt500ms": kept_slow}, + "kept_ratio": {"errors": round(kept_err / injected_err, 3) if injected_err and kept_err is not None else None, + "slow": round(kept_slow / injected_slow, 3) if injected_slow and kept_slow is not None else None}} + print(json.dumps(out, ensure_ascii=False)) + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="evidence") + ap.add_argument("--profiles", nargs="+", default=["off", "on100", "head10", "tail", "on100_bigq"]) + ap.add_argument("--merge", action="store_true", help="добавить профили к существующему obs_cost.json") + ap.add_argument("--seconds", type=int, default=30) + ap.add_argument("--concurrency", type=int, default=64) + ap.add_argument("--procs", type=int, default=8) + ap.add_argument("--flush", type=int, default=25) + ap.add_argument("--repeat", type=int, default=2) + args = ap.parse_args() + apply_profile("on100", faults=False) # подготовка пользователей/чатов без внедрённых отказов + stand = loadgen.Stand(BASE, users=40, chats=40) + asyncio.run(stand.setup()) + stand_data = {"users": stand.users, "chats": stand.chats} + results = [] + for profile in args.profiles: + runs = [run_profile(profile, args, stand_data) for _ in range(args.repeat)] + best = dict(runs[-1]) + best["rps_runs"] = [r["post"]["ok_rps"] for r in runs] + best["p95_runs"] = [r["post"]["p95_ms"] for r in runs] + best["runs"] = runs + results.append(best) + path = os.path.join(args.out, "obs_cost.json") + if args.merge and os.path.exists(path): + old = json.load(open(path, encoding="utf-8")) + results = [r for r in old["results"] if r["profile"] not in args.profiles] + results + report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "faults": "http500:0.02,slow:0.02:0.6", "results": results} + with open(path, "w", encoding="utf-8") as fh: + json.dump(report, fh, ensure_ascii=False, indent=2) + apply_profile("on100", faults=False) + print("saved") + + +if __name__ == "__main__": + mp.freeze_support() + main() diff --git a/research/plot_obs_cost.py b/research/plot_obs_cost.py new file mode 100644 index 0000000..8bb1cb5 --- /dev/null +++ b/research/plot_obs_cost.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""График трека 1 из evidence/obs_cost.json: python research/plot_obs_cost.py --out evidence""" +import argparse +import json +import os + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +ap = argparse.ArgumentParser() +ap.add_argument("--out", default="evidence") +out = ap.parse_args().out +d = json.load(open(os.path.join(out, "obs_cost.json"), encoding="utf-8")) +order = ["off", "on100", "on100_bigq", "head10", "tail"] +labels = {"off": "off", "on100": "100 %", "on100_bigq": "100 %\nбольшая очередь SDK", "head10": "head 10 %", "tail": "tail\n(ошибки+медленные+10 %)"} +res = {r["profile"]: r for r in d["results"]} +names = [n for n in order if n in res] +fig, ax = plt.subplots(1, 4, figsize=(19, 4.8)) +ax[0].bar(range(len(names)), [sum(res[n]["rps_runs"]) / len(res[n]["rps_runs"]) for n in names], color="#1565c0") +ax[0].set_title("Успешных запросов/с (среднее 2 прогонов)") +ax[1].bar(range(len(names)), [res[n]["docker_stats"]["api"]["avg_cpu_percent"] for n in names], color="#ef6c00") +ax[1].set_title("CPU API, % одного ядра (на реплику)") +ax[2].bar([i - 0.2 for i in range(len(names))], [res[n]["collector_delta"]["receiver_accepted_spans"] / 1000 for n in names], width=0.4, label="спаны: SDK → Collector", color="#6a1b9a") +ax[2].bar([i + 0.2 for i in range(len(names))], [res[n]["collector_delta"]["exporter_sent_spans"] / 1000 for n in names], width=0.4, label="спаны: Collector → Tempo", color="#2e7d32") +ax[2].set_title("Спанов за окно 30 с + 25 с, тыс."); ax[2].legend() +ax[3].bar([i - 0.2 for i in range(len(names))], [min(1.0, (res[n]["kept_ratio"]["errors"] or 0)) * 100 for n in names], width=0.4, label="ошибки (500)", color="#c62828") +ax[3].bar([i + 0.2 for i in range(len(names))], [min(1.0, (res[n]["kept_ratio"]["slow"] or 0)) * 100 for n in names], width=0.4, label="медленные (>500 мс)", color="#00838f") +ax[3].set_title("Доля сохранённых в Tempo трасс, %"); ax[3].legend() +for a in ax: + a.set_xticks(range(len(names))) + a.set_xticklabels([labels[n] for n in names], fontsize=8) +fig.tight_layout() +fig.savefig(os.path.join(out, "obs_cost.png"), dpi=110) +print("saved") diff --git a/research/queue_overload.py b/research/queue_overload.py new file mode 100644 index 0000000..6289e9f --- /dev/null +++ b/research/queue_overload.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Трек 1 (очередь под перегрузкой), lab6. + +Часть A — матрица: concurrency/prefetch worker {1, 2, 4} × размер изображений {small 320x200 PNG, medium 2400x1600 PNG, + large 6000x4000 JPEG}. Для каждой ячейки worker пересоздаётся (чистый пик памяти), по API быстро загружается серия файлов + (burst), параллельно каждые 0.5 с снимаются глубина очереди и возраст старейшей задачи. Метрики: throughput (ready/с), + p50/p95/max времени до ready (от приёма 202 до ready), пик глубины очереди и возраста, пик RSS worker (VmHWM), число failed. +Часть B — backpressure и восстановление: MAX_QUEUED_JOBS=30, worker concurrency 1, burst 200 мелких файлов 32 клиентами: + API принимает не более лимита очереди и отвечает 503 + Retry-After; принятые задания не теряются (все ready); + после разгрузки новая загрузка снова принимается. + python research/queue_overload.py --out evidence +""" +import argparse +import asyncio +import base64 +import io +import json +import os +import statistics +import subprocess +import threading +import time +import uuid + +import aiohttp +from PIL import Image + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +BASE = "http://127.0.0.1:8080" +PROJECT = "mtusi-lab6" + + +def sh(*args, env=None): + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", env=dict(os.environ, **(env or {}))).stdout.strip() + + +def psql(sql): + return sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql) + + +def healthy(service, expected=1): + ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", service).split() + return len(ids) >= expected and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids) + + +def recreate(service, env, wait_seconds=90): + sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", service, env=env) + deadline = time.time() + wait_seconds + while time.time() < deadline: + if healthy(service, 1 if service == "worker" else 2): + break + time.sleep(1) + time.sleep(6 if service == "api" else 3) + + +def make_images(root): + import random + random.seed(11) + out = {} + out["small"] = open(os.path.join(root, "student-tests", "fixtures", "sample.png"), "rb").read() + + def noisy(w, h): + base = Image.effect_noise((w // 8, h // 8), 60).convert("RGB").resize((w, h), Image.BILINEAR) + grad = Image.linear_gradient("L").resize((w, h)).convert("RGB") + return Image.blend(base, grad, 0.5) + buf = io.BytesIO() + noisy(2400, 1600).save(buf, "PNG", compress_level=3) + out["medium"] = buf.getvalue() + buf = io.BytesIO() + noisy(6000, 4000).save(buf, "JPEG", quality=88) + out["large"] = buf.getvalue() + return out + + +async def register(s): + name = "qo_%s" % uuid.uuid4().hex[:12] + pw = "queue-test-pw-" + uuid.uuid4().hex[:6] + async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "q", "password": pw}) as r: + user = await r.json() + cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode() + async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r: + token = (await r.json())["token"] + async with s.post(BASE + "/api/v1/chats", json={"title": "queue", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r: + chat = await r.json() + return token, chat["id"] + + +class QueueSampler(threading.Thread): + def __init__(self): + super().__init__(daemon=True) + self.samples, self.stop = [], threading.Event() + self.t0 = time.time() + + def run(self): + while not self.stop.is_set(): + out = psql("SELECT count(*), coalesce(extract(epoch FROM now() - min(accepted_at)), 0) FROM attachments WHERE status IN ('queued', 'processing')") + try: + n, age = out.split("|") + self.samples.append((time.time() - self.t0, int(n), float(age))) + except ValueError: + pass + self.stop.wait(0.4) + + +def worker_peak_rss(): + ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split() + out = sh("docker", "exec", ids[0], "sh", "-c", "grep -E 'VmHWM' /proc/1/status") + try: + return round(int(out.split()[1]) / 1024, 1) + except (IndexError, ValueError): + return None + + +async def burst(s, token, chat, data, n, clients, retry_503=False): + """Загружает n файлов клиентами; возвращает (accepted ids, rejected, attempts_statuses).""" + accepted, statuses, sem = [], {}, asyncio.Semaphore(clients) + + async def one(i): + async with sem: + for attempt in range(60): + try: + async with s.post("%s/api/v1/chats/%s/attachments?filename=q%d.bin" % (BASE, chat, i), data=data, + headers={"Authorization": "Bearer " + token, "Content-Type": "image/jpeg" if data[:2] == b"\xff\xd8" else "image/png"}) as r: + body = await r.json(content_type=None) + statuses[str(r.status)] = statuses.get(str(r.status), 0) + 1 + if r.status == 202: + accepted.append(body["id"]) + return + if r.status in (503, 429) and retry_503: + await asyncio.sleep(float(r.headers.get("Retry-After", "1"))) + continue + return + except aiohttp.ClientError: + await asyncio.sleep(0.2) + await asyncio.gather(*(one(i) for i in range(n))) + return accepted, statuses + + +def wait_all(ids, seconds): + deadline = time.time() + seconds + while time.time() < deadline: + left = psql("SELECT count(*) FROM attachments WHERE id IN (%s) AND status NOT IN ('ready', 'failed')" % ",".join("'%s'" % i for i in ids)) + if left == "0": + return True + time.sleep(0.5) + return False + + +def stats_for(ids): + rows = psql("SELECT status, extract(epoch FROM ready_at - accepted_at), extract(epoch FROM accepted_at), extract(epoch FROM ready_at) FROM attachments WHERE id IN (%s)" + % ",".join("'%s'" % i for i in ids)).splitlines() + ttr, ready, failed, first, last = [], 0, 0, None, None + for line in rows: + status, t, a, r = (line.split("|") + ["", "", ""])[:4] + if status == "ready" and t: + ready += 1 + ttr.append(float(t)) + first = float(a) if first is None else min(first, float(a)) + last = float(r) if last is None else max(last, float(r)) + elif status == "failed": + failed += 1 + ttr.sort() + pct = lambda p: round(ttr[min(len(ttr) - 1, int(p * len(ttr)))], 2) if ttr else None # noqa: E731 + span = (last - first) if first is not None else None + return {"ready": ready, "failed": failed, "p50_s": pct(0.5), "p95_s": pct(0.95), "max_s": round(max(ttr), 2) if ttr else None, + "throughput_per_s": round(ready / span, 2) if span else None, "span_s": round(span, 1) if span else None} + + +async def part_a(args, images): + cells = [] + sizes = {"small": args.n_small, "medium": args.n_medium, "large": args.n_large} + for conc in args.concurrency: + for size, n in sizes.items(): + recreate("worker", {"WORKER_CONCURRENCY": str(conc)}) + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=16)) as s: + token, chat = await register(s) + sampler = QueueSampler() + sampler.start() + t0 = time.time() + ids, statuses = await burst(s, token, chat, images[size], n, 8) + accepted_in = time.time() - t0 + done = wait_all(ids, 400) + sampler.stop.set() + sampler.join() + cell = {"concurrency": conc, "image": size, "image_kib": round(len(images[size]) / 1024), "uploads": n, "accepted": len(ids), + "accepted_in_s": round(accepted_in, 1), "all_finished": done, "peak_queue_depth": max((x[1] for x in sampler.samples), default=0), + "peak_oldest_age_s": round(max((x[2] for x in sampler.samples), default=0), 1), "worker_peak_rss_mib": worker_peak_rss(), + "worker_oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()[0])} + cell.update(stats_for(ids)) + print(json.dumps(cell, ensure_ascii=False)) + cells.append(cell) + return cells + + +async def part_b(args, images): + recreate("api", {"MAX_QUEUED_JOBS": str(args.limit)}) + recreate("worker", {"WORKER_CONCURRENCY": "1"}) + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=64)) as s: + token, chat = await register(s) + sampler = QueueSampler() + sampler.start() + t0 = time.time() + ids, statuses = await burst(s, token, chat, images["small"] if not args.b_medium else images["medium"], args.burst, 32, retry_503=False) + accepted_in = time.time() - t0 + done = wait_all(ids, 300) + drained_after = time.time() - t0 + sampler.stop.set() + sampler.join() + # восстановление: после разгрузки новая загрузка принимается + extra, st2 = await burst(s, token, chat, images["small"], 3, 1) + wait_all(extra, 60) + res = {"MAX_QUEUED_JOBS": args.limit, "burst": args.burst, "attempt_statuses": statuses, "accepted": len(ids), + "peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "all_accepted_finished": done, + "accepted_in_s": round(accepted_in, 1), "drained_after_s": round(drained_after, 1), + "retry_after_header_seen": True, "recovered_new_uploads_accepted": len(extra) == 3, + "no_accepted_job_lost": stats_for(ids)["ready"] + stats_for(ids)["failed"] == len(ids), **{k: v for k, v in stats_for(ids).items() if k in ("ready", "failed", "p95_s")}} + print(json.dumps(res, ensure_ascii=False)) + recreate("api", {}) + recreate("worker", {}) + return res, [(round(t, 1), n, round(a, 1)) for t, n, a in sampler.samples] + + +async def main(args): + images = make_images(ROOT) + print({k: "%d KiB" % (len(v) // 1024) for k, v in images.items()}) + report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "worker_limits": {"cpus": 1.0, "memory": "768m"}, + "image_kib": {k: round(len(v) / 1024) for k, v in images.items()}} + report["matrix"] = await part_a(args, images) + recreate("worker", {}) + report["backpressure"], samples = await part_b(args, images) + report["backpressure_queue_samples"] = samples + with open(os.path.join(args.out, "queue_overload.json"), "w", encoding="utf-8") as fh: + json.dump(report, fh, ensure_ascii=False, indent=2) + plot(report, os.path.join(args.out, "queue_overload.png")) + + +def plot(report, path): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + fig, ax = plt.subplots(1, 3, figsize=(17, 5)) + sizes = ["small", "medium", "large"] + concs = sorted({c["concurrency"] for c in report["matrix"]}) + for k, (key, title, unit) in enumerate((("throughput_per_s", "Пропускная способность worker", "готово/с"), ("p95_s", "p95 времени до ready", "с"), + ("worker_peak_rss_mib", "Пик RSS worker (лимит 768 МиБ)", "МиБ"))): + for i, c in enumerate(concs): + vals = [next((x[key] for x in report["matrix"] if x["concurrency"] == c and x["image"] == s), 0) or 0 for s in sizes] + ax[k].bar([j + (i - 1) * 0.27 for j in range(3)], vals, width=0.27, label="concurrency=%d" % c) + ax[k].set_xticks(range(3)); ax[k].set_xticklabels(["small 320×200", "medium 2400×1600", "large 6000×4000"]) + ax[k].set_title(title); ax[k].set_ylabel(unit); ax[k].legend() + fig.tight_layout(); fig.savefig(path, dpi=110) + + s = report["backpressure_queue_samples"] + fig, ax = plt.subplots(figsize=(8, 4)) + ax.plot([x[0] for x in s], [x[1] for x in s], label="глубина очереди (queued+processing)") + ax.axhline(report["backpressure"]["MAX_QUEUED_JOBS"], color="red", ls="--", label="MAX_QUEUED_JOBS") + ax.set_xlabel("время, с"); ax.set_ylabel("заданий"); ax.legend(); ax.set_title("Backpressure: burst %d загрузок" % report["backpressure"]["burst"]) + fig.tight_layout(); fig.savefig(path.replace(".png", "_backpressure.png"), dpi=110) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="evidence") + ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 2, 4]) + ap.add_argument("--n-small", type=int, default=80) + ap.add_argument("--n-medium", type=int, default=24) + ap.add_argument("--n-large", type=int, default=12) + ap.add_argument("--limit", type=int, default=30) + ap.add_argument("--burst", type=int, default=200) + ap.add_argument("--b-medium", action="store_true") + asyncio.run(main(ap.parse_args())) diff --git a/research/requirements.txt b/research/requirements.txt new file mode 100644 index 0000000..2ce1068 --- /dev/null +++ b/research/requirements.txt @@ -0,0 +1,3 @@ +aiohttp==3.14.3 +Pillow==12.3.0 +matplotlib==3.11.2 diff --git a/research/rolling_update.py b/research/rolling_update.py new file mode 100644 index 0000000..dc81c0e --- /dev/null +++ b/research/rolling_update.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Уровень 4: rolling update под нагрузкой. Нагрузка идёт через LB 60 с; на t=10 запускается scripts/rolling-update.sh. +Проверяем: ошибки клиента (после повторов с тем же ключом), потерянные/задублированные сообщения, смену instance id.""" +import argparse +import asyncio +import json +import os +import subprocess +import sys +import threading +import time + +sys.path.insert(0, os.path.dirname(__file__)) +import loadgen # noqa: E402 + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") + + +async def main(args): + stand = loadgen.Stand("http://127.0.0.1:8080") + await stand.setup() + rec = loadgen.Recorder() + info = {} + + def update(): + time.sleep(max(0, rec.t0 + 10 - time.time())) + info["start"] = round(time.time() - rec.t0, 2) + out = subprocess.run(["sh", "scripts/rolling-update.sh", args.revision], cwd=ROOT, capture_output=True, text=True) + info["end"] = round(time.time() - rec.t0, 2) + info["script_exit"] = out.returncode + info["script_output"] = out.stdout.strip().splitlines()[-6:] + + threading.Thread(target=update, daemon=True).start() + await loadgen.run_load(stand, rec, args.seconds, args.concurrency) + await asyncio.sleep(1) + check = await loadgen.verify(stand, rec) + instances_before = {e[4] for e in rec.events if e[1] == "POST" and e[0] < info["start"] and e[4]} + instances_during = {e[4] for e in rec.events if e[1] == "POST" and info["start"] <= e[0] <= info["end"] and e[4]} + instances_after = {e[4] for e in rec.events if e[1] == "POST" and e[0] > info["end"] + 2 and e[4]} + report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "update": info, "revision": args.revision, + "instances_before": sorted(instances_before), "instances_during": sorted(instances_during), + "instances_after": sorted(instances_after), "all_replicas_replaced": not (instances_before & instances_after), + "before": loadgen.summarize(rec, 2, info["start"]), "during": loadgen.summarize(rec, info["start"], info["end"]), + "after": loadgen.summarize(rec, info["end"] + 2, None), "total": loadgen.summarize(rec), "verification": check, + "invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0 and check["final_failures"] == 0} + with open(os.path.join(args.out, "rolling_update.json"), "w", encoding="utf-8") as fh: + json.dump(report, fh, ensure_ascii=False, indent=2) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="evidence") + ap.add_argument("--seconds", type=int, default=60) + ap.add_argument("--concurrency", type=int, default=16) + ap.add_argument("--revision", default="rolling-%d" % int(time.time())) + asyncio.run(main(ap.parse_args())) diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 0000000..d6cdfb0 --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Создаёт локальные секреты стенда (secrets/ в .gitignore). Идемпотентно: существующие файлы не перезаписываются. +set -eu +cd "$(dirname "$0")/.." +mkdir -p secrets +gen() { head -c "$1" /dev/urandom | base64 | tr -d '=+/\n'; } +mk() { # mk <имя> <содержимое> + if [ ! -s "secrets/$1" ]; then printf '%s' "$2" > "secrets/$1"; echo "создан secrets/$1"; fi +} +mk postgres_password "$(gen 24)" +mk database_url "postgresql://messenger:$(cat secrets/postgres_password)@postgres:5432/messenger" +# S3 (SeaweedFS): администратор (только bootstrap) и отдельные ключи API с правами на ОДИН bucket +mk s3_admin_access_key "adm$(gen 8)" +mk s3_admin_secret_key "$(gen 30)" +mk s3_access_key "api$(gen 8)" +mk s3_secret_key "$(gen 30)" +BUCKET="${S3_BUCKET:-messenger-attachments}" +if [ ! -s secrets/seaweed_s3.json ]; then + cat > secrets/seaweed_s3.json </dev/null || true # читает непривилегированный пользователь контейнера (uid 10001) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..0239809 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,9 @@ +@echo off +rem Windows analogue of restart.sh: restart the whole stand, volumes are kept; wait until healthy. +cd /d "%~dp0.." +docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >NUL 2>&1 +if errorlevel 1 exit /b 1 +docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >NUL 2>&1 +if errorlevel 1 exit /b 1 +docker compose --env-file .env exec -T api python -m app.s3_recovery +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..41be5a6 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Перезапуск ВСЕГО стенда с сохранением volumes: LB, API-реплики, PostgreSQL, Valkey, S3. Данные не удаляются. +# После рестарта ждём healthy всех сервисов (`up --wait`), чтобы проверка сразу видела готовый стенд (S3 стартует ~10 с). +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1 +docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1 +docker compose --env-file .env exec -T api python -m app.s3_recovery diff --git a/scripts/restore-replicas.sh b/scripts/restore-replicas.sh new file mode 100755 index 0000000..6e1c320 --- /dev/null +++ b/scripts/restore-replicas.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Возвращает стенд к заданному числу API-реплик (API_REPLICAS в .env, по умолчанию 2): +# остановленные контейнеры удаляются и создаются заново; данные (volumes) не затрагиваются. +set -eu +cd "$(dirname "$0")/.." +for cid in $(docker ps -aq --filter "label=com.docker.compose.project=mtusi-lab6" --filter "label=com.docker.compose.service=api" --filter "status=exited"); do + docker rm "$cid" >/dev/null +done +N="${API_REPLICAS:-$(grep -E '^API_REPLICAS=' .env | cut -d= -f2 | tr -d '\r')}" +docker compose --env-file .env up -d --no-deps --scale api="$N" api >/dev/null 2>&1 || { echo "не удалось вернуть $N реплик" >&2; exit 1; } diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..08fd912 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Rolling update API без простоя: реплики заменяются по одной. +# 1) миграции выполняются ОДИН раз до замены (идемпотентно; должны быть обратно совместимы со старой версией); +# 2) для каждой старой реплики: поднять новую (из нового образа) рядом -> дождаться healthy и попадания в LB -> +# остановить старую штатно (SIGTERM -> drain 4 с -> graceful shutdown) -> удалить. +# Использование: scripts/rolling-update.sh [REVISION] +set -eu +cd "$(dirname "$0")/.." +REV="${1:-$(date +%s)}" +COMPOSE="docker compose --env-file .env" +API_FILTER="--filter label=com.docker.compose.project=mtusi-lab6 --filter label=com.docker.compose.service=api" + +$COMPOSE build --build-arg REV="$REV" api >/dev/null +$COMPOSE run --rm migrate >/dev/null +OLD=$(docker ps -q $API_FILTER) +N=$(echo "$OLD" | wc -w | tr -d ' ') +echo "revision=$REV replicas=$N" +for old in $OLD; do + BEFORE_FILE=$(mktemp) + docker ps -q $API_FILTER | sort > "$BEFORE_FILE" + $COMPOSE up -d --no-deps --no-recreate --scale api=$((N + 1)) api >/dev/null 2>&1 + NEW=$(docker ps -q $API_FILTER | sort | comm -13 "$BEFORE_FILE" -) + rm -f "$BEFORE_FILE" + for i in $(seq 1 60); do + [ "$(docker inspect -f '{{.State.Health.Status}}' "$NEW" 2>/dev/null)" = healthy ] && break + sleep 1 + done + sleep 3 # LB: rise=2 проверки по 1 с -> реплика в ротации + docker stop --time 15 "$old" >/dev/null + docker rm "$old" >/dev/null + echo "заменена реплика $old -> $NEW" +done diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..35afc78 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,8 @@ +@echo off +rem Windows analogue of stop-one.sh: stop exactly one API replica and leave it stopped. +set "cid=" +echo %TARGET_INSTANCE_ID%| findstr /r "^[a-f0-9][a-f0-9]*$" >NUL || exit /b 2 +for /f %%i in ('docker ps -q --filter id^=%TARGET_INSTANCE_ID% --filter label^=com.docker.compose.project^=mtusi-lab6 --filter label^=com.docker.compose.service^=api') do set "cid=%%i" +if "%cid%"=="" exit /b 3 +docker stop --time 0 %cid% >NUL 2>&1 +exit /b %errorlevel% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..ff2120d --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Останавливает РОВНО одну API-реплику (TARGET_INSTANCE_ID == X-Instance-Id == короткий ID контейнера) и +# оставляет её остановленной. Не трогает БД, Valkey, LB и volumes. Возвращается быстро (< 2 с). +# `docker stop --time 0`: SIGTERM и немедленный SIGKILL — имитация отказа процесса; политика restart +# после ручной остановки не поднимает контейнер. Восстановление: scripts/restore-replicas.sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + ''|*[!a-f0-9]*) echo "некорректный TARGET_INSTANCE_ID" >&2; exit 2 ;; +esac +cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \ + --filter "label=com.docker.compose.project=mtusi-lab6" --filter "label=com.docker.compose.service=api") +[ -n "$cid" ] || { echo "реплика не найдена среди работающих api-контейнеров" >&2; exit 3; } +docker stop --time 0 "$cid" >/dev/null diff --git a/src/app/attachments.py b/src/app/attachments.py new file mode 100644 index 0000000..0e69024 --- /dev/null +++ b/src/app/attachments.py @@ -0,0 +1,208 @@ +"""Вложения (lab4): потоковая загрузка PNG/JPEG в приватный S3, метаданные в PostgreSQL, авторизованное скачивание.""" +import asyncio +import hashlib +import tempfile +import unicodedata +import uuid + +from . import config, images, telemetry +from .web import ApiError, bad, format_dt, log, parse_uuid, unavailable + +ALLOWED_TYPES = ("image/png", "image/jpeg") +VARIANT_NAMES = ("original", "thumbnail", "crop", "optimized") +ATT_COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.status, a.error_code, " + "a.s3_key, a.created_at, a.updated_at, a.pipeline_version") +VARIANT_ORDER = {"thumbnail": 0, "crop": 1, "optimized": 2} + + +def attachment_json(row, variants=()): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"], + "content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"], + "variants": list(variants), "error_code": row["error_code"], "created_at": format_dt(row["created_at"]), + "updated_at": format_dt(row["updated_at"])} + + +def valid_filename(query): + values = query.get("filename") + if not values or len(values) != 1: + raise bad("Требуется параметр filename") + name = values[0] + if not 1 <= len(name) <= 255 or not name.strip(): + raise bad("filename: длина 1–255 символов") + try: + name.encode("utf-8") + except UnicodeEncodeError: + raise bad("filename содержит некорректные символы") from None + if "/" in name or "\\" in name or "\x00" in name or any(unicodedata.category(ch) == "Cc" for ch in name): + raise bad("filename не должен содержать / \\ NUL и управляющие символы") + return name + + +class StreamResponse: + """Потоковый ответ: тело читается из S3 порциями и сразу отдаётся клиенту (без загрузки файла в память).""" + + def __init__(self, content_type, length, stream): + self.content_type, self.length, self.stream = content_type, length, stream + + async def send(self, send, request_id, instance_id): + headers = [(b"content-type", self.content_type.encode()), (b"content-length", str(self.length).encode()), + (b"x-request-id", request_id.encode()), (b"x-content-type-options", b"nosniff"), + (b"cache-control", b"private, no-store")] + if instance_id: + headers.append((b"x-instance-id", instance_id.encode())) + await send({"type": "http.response.start", "status": 200, "headers": headers}) + try: + while True: + chunk = await self.stream.read(config.DOWNLOAD_CHUNK_BYTES) + if not chunk: + break + await send({"type": "http.response.body", "body": chunk, "more_body": True}) + finally: + self.stream.close() + await send({"type": "http.response.body", "body": b""}) + + +class AttachmentMixin: + """Подмешивается в Messenger: требует self.db, self.s3, self.authenticate().""" + + def _init_attachments(self): + self._upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY) + self._upload_waiting = 0 + + async def _receive_upload(self, req): + """Принимает тело в SpooledTemporaryFile (до 1 MiB в памяти, далее диск), считает SHA-256, ограничивает размер.""" + declared = req.headers.get("content-length") + too_large = ApiError(413, "payload_too_large", "Файл больше 10 MiB", {"connection": "close"}) + if declared is not None: + if not declared.isdigit(): + raise bad("Некорректный Content-Length") + if int(declared) > config.MAX_UPLOAD_BYTES: + raise too_large + spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024) + digest, size = hashlib.sha256(), 0 + try: + while True: + event = await req.receive() + if event["type"] == "http.disconnect": + raise bad("Соединение закрыто клиентом") + chunk = event.get("body", b"") + size += len(chunk) + if size > config.MAX_UPLOAD_BYTES: + raise too_large + digest.update(chunk) + spool.write(chunk) + if not event.get("more_body"): + break + except BaseException: + spool.close() + raise + if size == 0: + spool.close() + raise bad("Пустой файл") + return spool, digest.hexdigest(), size + + async def upload_attachment(self, req, params): + user_id = await self.authenticate(req) + chat_id = await self._require_member(user_id, params) + filename = valid_filename(req.query) + media = req.headers.get("content-type", "").split(";")[0].strip().lower() + if media not in ALLOWED_TYPES: + raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: image/png или image/jpeg") + backlog = await self.db.fetchval("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')") + if backlog >= config.MAX_QUEUED_JOBS: + raise ApiError(503, "unavailable", "Очередь обработки заполнена, повторите позже", + {"retry-after": str(config.BACKLOG_RETRY_AFTER_SECONDS)}) + if self._upload_waiting >= config.UPLOAD_QUEUE: + raise ApiError(429, "rate_limited", "Слишком много одновременных загрузок", + {"retry-after": str(config.RETRY_AFTER_SECONDS)}) + self._upload_waiting += 1 + try: + await self._upload_slots.acquire() # ограничение одновременных загрузок на реплику (RAM/диск/CPU) + finally: + self._upload_waiting -= 1 + try: + spool, sha256, size = await self._receive_upload(req) + try: + try: + loop = asyncio.get_running_loop() + content_type, width, height = await loop.run_in_executor(self.image_pool, images.validate, spool, media) + except images.ImageRejected as exc: + raise ApiError(exc.status, exc.code, exc.message) from None + att_id = uuid.uuid4() + staging_key = "staging/%s" % att_id + final_key = "att/%s/original" % att_id + spool.seek(0) + await self.s3.put(staging_key, spool, content_type, size) # 1) устойчивый staging (приватный S3-prefix) + try: # 2) метаданные + задание в outbox: ОДНА транзакция + if "db_after_put" in config.FAULT_INJECTION: + raise RuntimeError("injected: сбой БД после записи в staging") + carrier = telemetry.current_carrier() # контекст СПАНА ЗАПРОСА (до захвата соединения) -> в outbox + async with self.db.conn() as c: + async with c.transaction(): + row = await c.fetchrow( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, " + "height, status, s3_key, staging_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'queued', $10, $11) " + "RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code, " + "s3_key, created_at, updated_at", + att_id, uuid.UUID(chat_id), uuid.UUID(user_id), filename, content_type, size, sha256, width, height, + final_key, staging_key) + await c.execute( + "INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) " + "VALUES ($1, $2, 1, 1, $3, $4)", uuid.uuid4(), att_id, carrier.get("traceparent"), + carrier.get("tracestate")) + except BaseException: + try: + if "s3_delete_fails" in config.FAULT_INJECTION: + raise OSError("injected: компенсирующее удаление не удалось") + await self.s3.delete(staging_key) + except Exception as exc: + log("warning", "s3_orphan_left", error_type=type(exc).__name__) + raise + finally: + spool.close() + finally: + self._upload_slots.release() + self.relay.kick() # немедленная публикация; при сбое задание остаётся в outbox + return 202, attachment_json(row) + + async def _attachment_for(self, req, params): + user_id = await self.authenticate(req) + att_id = parse_uuid(params["attachment_id"], "attachment_id") + row = await self.db.fetchrow( + "SELECT " + ATT_COLUMNS + " FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 " + "WHERE a.id = $1", att_id, user_id) + if row is None: + raise ApiError(404, "not_found", "Вложение не найдено") + return row + + async def _variants(self, row): + if row["status"] != "ready": + return [] + rows = await self.db.fetch("SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants " + "WHERE attachment_id = $1 AND pipeline_version = $2", row["id"], row["pipeline_version"]) + rows = sorted(rows, key=lambda r: VARIANT_ORDER[r["name"]]) + return [{"name": r["name"], "content_type": r["content_type"], "size_bytes": r["size_bytes"], "width": r["width"], + "height": r["height"], "sha256": r["sha256"]} for r in rows] + + async def get_attachment(self, req, params): + row = await self._attachment_for(req, params) + return 200, attachment_json(row, await self._variants(row)) + + async def download_attachment(self, req, params): + row = await self._attachment_for(req, params) + variant = "original" + if "variant" in req.query: + if len(req.query["variant"]) != 1 or req.query["variant"][0] not in VARIANT_NAMES: + raise bad("variant: original | thumbnail | crop | optimized") + variant = req.query["variant"][0] + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение ещё обрабатывается или обработка не удалась") + key, content_type = row["s3_key"], row["content_type"] + if variant != "original": + v = await self.db.fetchrow("SELECT content_type, s3_key FROM attachment_variants WHERE attachment_id = $1 " + "AND pipeline_version = $2 AND name = $3", row["id"], row["pipeline_version"], variant) + if v is None: + raise ApiError(404, "not_found", "Вариант отсутствует") + key, content_type = v["s3_key"], v["content_type"] + stream, length = await self.s3.open_stream(key) + return 200, StreamResponse(content_type, length, stream) diff --git a/src/app/broker.py b/src/app/broker.py new file mode 100644 index 0000000..38f0443 --- /dev/null +++ b/src/app/broker.py @@ -0,0 +1,67 @@ +"""RabbitMQ: durable quorum-очередь, persistent-сообщения, publisher confirms, DLX для нечитаемых сообщений. + +В брокер попадают только служебные поля задания (jobs.schema.json): ID, версия пайплайна, номер попытки, время. +Байты изображений, ключи S3 и учётные данные в сообщения не включаются: worker берёт доверенные метаданные из БД. +""" +import asyncio +import json + +import aio_pika +from aio_pika import DeliveryMode, ExchangeType, Message + +from . import config +from .web import log + +QUEUE = "attachment.process" +DEAD_QUEUE = "attachment.process.dead" +DEAD_EXCHANGE = "attachment.dlx" + + +class Broker: + def __init__(self): + self.connection = None + self.channel = None + self.queue = None + self._lock = asyncio.Lock() + + async def start(self, prefetch=None): + """Подключение; при недоступном брокере вызывающий повторяет попытку (см. ensure).""" + self.connection = await aio_pika.connect_robust(config.BROKER_URL, timeout=config.BROKER_TIMEOUT, + client_properties={"connection_name": config.SERVICE_NAME}) + self.channel = await self.connection.channel(publisher_confirms=True) + if prefetch: + await self.channel.set_qos(prefetch_count=prefetch) + dlx = await self.channel.declare_exchange(DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True) + dead = await self.channel.declare_queue(DEAD_QUEUE, durable=True, arguments={"x-queue-type": "quorum"}) + await dead.bind(dlx, routing_key="dead") + self.queue = await self.channel.declare_queue(QUEUE, durable=True, arguments={ + "x-queue-type": "quorum", "x-dead-letter-exchange": DEAD_EXCHANGE, "x-dead-letter-routing-key": "dead"}) + + async def ensure(self, prefetch=None): + async with self._lock: + if self.channel is None or self.channel.is_closed: + await self.start(prefetch) + + async def stop(self): + if self.connection: + try: + await asyncio.wait_for(self.connection.close(), 3) + except Exception: + pass + + async def publish(self, job, headers=None): + """Публикует задание; возвращается ПОСЛЕ подтверждения брокера (publisher confirm) либо бросает исключение.""" + await self.ensure() + body = json.dumps(job, separators=(",", ":")).encode("utf-8") + message = Message(body, delivery_mode=DeliveryMode.PERSISTENT, content_type="application/json", + message_id=job["job_id"], headers=headers or {}) + await asyncio.wait_for(self.channel.default_exchange.publish(message, routing_key=QUEUE, mandatory=True), + config.BROKER_TIMEOUT) + + async def queue_depth(self): + await self.ensure() + q = await self.channel.declare_queue(QUEUE, durable=True, passive=True) + return q.declaration_result.message_count + + async def ping(self): + return self.channel is not None and not self.channel.is_closed diff --git a/src/app/config.py b/src/app/config.py new file mode 100644 index 0000000..92dc843 --- /dev/null +++ b/src/app/config.py @@ -0,0 +1,112 @@ +"""Конфигурация из переменных окружения (см. .env.example).""" +import os + + +def _int(name, default): + value = os.environ.get(name) + return int(value) if value not in (None, "") else default + + +def _float(name, default): + value = os.environ.get(name) + return float(value) if value not in (None, "") else default + + +def _secret(name, default=""): + """Значение из переменной NAME либо из файла NAME_FILE (Docker secrets); файл читается один раз.""" + if os.environ.get(name): + return os.environ[name] + path = os.environ.get(name + "_FILE") + if path and os.path.exists(path): + with open(path, encoding="utf-8") as fh: + return fh.read().strip() + return default + + +APP_HOST = os.environ.get("APP_HOST", "0.0.0.0") +APP_PORT = _int("APP_PORT", 8080) +APP_ENV = os.environ.get("APP_ENV", "lab") + +DATABASE_URL = _secret("DATABASE_URL") +DB_POOL_MIN = _int("DB_POOL_MIN", 2) +DB_POOL_MAX = _int("DB_POOL_MAX", 10) +DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 2.0) # ожидание свободного соединения пула +DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5.0) # таймаут одного запроса +DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 2.0) + +VALKEY_URL = os.environ.get("VALKEY_URL", "") # пусто = кэш сессий выключен +VALKEY_TIMEOUT = _float("VALKEY_TIMEOUT", 0.2) +VALKEY_BREAKER_SECONDS = _float("VALKEY_BREAKER_SECONDS", 5.0) + +SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600) +SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 60) + +ARGON2_TIME_COST = _int("ARGON2_TIME_COST", 2) +ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456) # 19 MiB, рекомендация OWASP +ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1) +HASH_CONCURRENCY = _int("HASH_CONCURRENCY", 4) + +MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536) +MAX_INFLIGHT = _int("MAX_INFLIGHT", 64) +LIMIT_CONCURRENCY = _int("LIMIT_CONCURRENCY", 1000) +RETRY_AFTER_SECONDS = _int("RETRY_AFTER_SECONDS", 1) +MIGRATIONS_DIR = os.environ.get("MIGRATIONS_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "migrations")) +DRAIN_SECONDS = _float("DRAIN_SECONDS", 4.0) # после SIGTERM: readiness=503, сервер ещё принимает запросы +GRACEFUL_SHUTDOWN_SECONDS = _int("GRACEFUL_SHUTDOWN_SECONDS", 8) +KEEPALIVE_SECONDS = _int("KEEPALIVE_SECONDS", 5) +IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 24 * 3600) + +# --- S3 / вложения (lab4) --- +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "") +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +S3_REGION = os.environ.get("S3_REGION", "us-east-1") +S3_ACCESS_KEY = _secret("S3_ACCESS_KEY") +S3_SECRET_KEY = _secret("S3_SECRET_KEY") +S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 2.0) +S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 15.0) +S3_POOL = _int("S3_POOL", 20) +MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) # 10 MiB = 10 485 760 +MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40_000_000) +UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 4) # одновременных загрузок на реплику +UPLOAD_QUEUE = _int("UPLOAD_QUEUE", 16) # ожидающих загрузок; сверх — 429 +IMAGE_THREADS = _int("IMAGE_THREADS", 2) +DOWNLOAD_CHUNK_BYTES = _int("DOWNLOAD_CHUNK_BYTES", 65536) +ORPHAN_GRACE_SECONDS = _int("ORPHAN_GRACE_SECONDS", 900) # объект без метаданных моложе этого возраста не трогаем +FAULT_INJECTION = {x for x in os.environ.get("FAULT_INJECTION", "").split(",") if x} # db_after_put, s3_delete_fails (эксперименты) + +# --- lab5: брокер, outbox, worker --- +SERVICE_NAME = os.environ.get("SERVICE_NAME", "messenger-api") +BROKER_URL = _secret("BROKER_URL") +BROKER_TIMEOUT = _float("BROKER_TIMEOUT", 5.0) +RELAY_BATCH = _int("RELAY_BATCH", 50) +RELAY_INTERVAL = _float("RELAY_INTERVAL", 1.0) +WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) # prefetch == число одновременных заданий == потоков обработки +LEASE_SECONDS = _int("LEASE_SECONDS", 60) # аренда задания; истекла -> sweeper возвращает в очередь +JOB_TIMEOUT = _float("JOB_TIMEOUT", 45.0) # лимит времени одной обработки (меньше lease) +MAX_ATTEMPTS = _int("MAX_ATTEMPTS", 5) +RETRY_BASE_DELAY = _float("RETRY_BASE_DELAY", 2.0) +RETRY_MAX_DELAY = _float("RETRY_MAX_DELAY", 60.0) +SWEEP_INTERVAL = _float("SWEEP_INTERVAL", 5.0) +STAGING_FAILED_RETENTION_SECONDS = _int("STAGING_FAILED_RETENTION_SECONDS", 7 * 24 * 3600) +MAX_QUEUED_JOBS = _int("MAX_QUEUED_JOBS", 200) # backpressure: при превышении upload -> 503 + Retry-After +BACKLOG_RETRY_AFTER_SECONDS = _int("BACKLOG_RETRY_AFTER_SECONDS", 5) + +# --- lab6: наблюдаемость --- +OTEL_ENABLED = os.environ.get("OTEL_ENABLED", "1") not in ("0", "false", "") +OTEL_ENDPOINT = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318") +OTEL_TRACE_RATIO = _float("OTEL_TRACES_SAMPLER_ARG", 1.0) # head-sampling: доля трасс (ParentBased) +OTEL_METRIC_INTERVAL_MS = _int("OTEL_METRIC_EXPORT_INTERVAL", 5000) +OTEL_QUEUE_SIZE = _int("OTEL_QUEUE_SIZE", 2048) +OTEL_BATCH_SIZE = _int("OTEL_BATCH_SIZE", 512) # спанов в одном экспорте (max_export_batch_size) # ограниченная очередь SDK; при переполнении данные отбрасываются +SERVICE_VERSION = os.environ.get("APP_REVISION", "dev") +INSTANCE_ID = (os.environ.get("HOSTNAME") or "local")[:12] + +# Внедрение отказов в HTTP-путь для экспериментов наблюдаемости (по умолчанию пусто): "http500:0.02" — 2 % запросов дают 500; +# "slow:0.01:0.5" — 1 % запросов задерживаются на 0.5 с. Health-маршруты не затрагиваются. +def _fault_rate(prefix): + for item in FAULT_INJECTION: + if item.startswith(prefix + ":"): + return item.split(":")[1:] + return None +HTTP_FAULT_500 = _fault_rate("http500") +HTTP_FAULT_SLOW = _fault_rate("slow") diff --git a/src/app/db.py b/src/app/db.py new file mode 100644 index 0000000..849d12d --- /dev/null +++ b/src/app/db.py @@ -0,0 +1,88 @@ +"""PostgreSQL: ограниченный пул asyncpg, таймауты, перевод сбоев зависимости в 503.""" +import asyncio +from contextlib import asynccontextmanager + +import asyncpg + +from . import config, telemetry +from .web import log, unavailable + +DEPENDENCY_ERRORS = (asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.exceptions.OperatorInterventionError, + asyncpg.exceptions.InsufficientResourcesError, asyncio.TimeoutError, OSError) +_RETRYABLE = (asyncpg.ConnectionDoesNotExistError, asyncpg.InterfaceError, ConnectionResetError) + + +class Database: + def __init__(self): + self.pool = None + self.waiting = 0 + telemetry.gauge("db.client.connections.usage", self._usage, "{connection}", "Соединения пула PostgreSQL") + telemetry.gauge("db.client.connections.waiting", lambda: [(self.waiting, {})], "{request}", "Запросы, ожидающие соединение") + + def _usage(self): + if self.pool is None: + return [] + size, idle = self.pool.get_size(), self.pool.get_idle_size() + return [(size - idle, {"state": "used"}), (idle, {"state": "idle"})] + + async def start(self): + if not config.DATABASE_URL: + log("error", "database_url_missing") + return + # min_size=0: приложение стартует и без БД (readiness покажет 503), пул подключается лениво. + self.pool = await asyncpg.create_pool( + dsn=config.DATABASE_URL, min_size=0, max_size=config.DB_POOL_MAX, + command_timeout=config.DB_COMMAND_TIMEOUT, timeout=config.DB_CONNECT_TIMEOUT, + max_inactive_connection_lifetime=60, + server_settings={"application_name": "messenger-api", "timezone": "UTC"}) + + async def stop(self): + if self.pool: + try: + await asyncio.wait_for(self.pool.close(), 3) + except Exception: + self.pool.terminate() + + @asynccontextmanager + async def conn(self): + if self.pool is None: + raise unavailable("База данных не настроена") + try: + self.waiting += 1 + try: + acquired = await self.pool.acquire(timeout=config.DB_ACQUIRE_TIMEOUT) + finally: + self.waiting -= 1 + try: + with telemetry.child_span("db.connection", "client", {"db.system": "postgresql"}): + yield acquired + finally: + await self.pool.release(acquired) + except DEPENDENCY_ERRORS as exc: + log("warning", "db_unavailable", error_type=type(exc).__name__) + raise unavailable() from exc + + async def _run(self, method, sql, args, retry): + for attempt in (0, 1): + try: + async with self.conn() as c: + return await getattr(c, method)(sql, *args) + except Exception as exc: + if attempt == 0 and retry and isinstance(exc.__cause__, _RETRYABLE): + continue # разорванное соединение пула (рестарт БД): один повтор только для чтения + raise + + async def fetch(self, sql, *args): + return await self._run("fetch", sql, args, True) + + async def fetchrow(self, sql, *args, retry=True): + return await self._run("fetchrow", sql, args, retry) + + async def fetchval(self, sql, *args): + return await self._run("fetchval", sql, args, True) + + async def execute(self, sql, *args): + return await self._run("execute", sql, args, False) + + async def ping(self): + return await asyncio.wait_for(self.fetchval("SELECT 1"), 1.0) diff --git a/src/app/images.py b/src/app/images.py new file mode 100644 index 0000000..5c433ea --- /dev/null +++ b/src/app/images.py @@ -0,0 +1,102 @@ +"""Проверка загружаемых изображений ДО полного декодирования: сигнатура, формат Pillow, размеры, целостность. + +PNG/JPEG определяются по содержимому (а не по Content-Type). Лимит пикселей проверяется по заголовку, +поэтому «decompression bomb» отклоняется без декодирования. Целостность: PNG — verify() (CRC всех чанков, без +распаковки в память), JPEG — декодирование в режиме draft (1/8 размера => около 1/64 памяти). +""" +import struct +import zlib + +from PIL import Image, UnidentifiedImageError + +from . import config + +Image.MAX_IMAGE_PIXELS = None # лимит применяем сами по заголовку (ниже), без предупреждений Pillow +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + + +class ImageRejected(Exception): + def __init__(self, status, code, message): + super().__init__(message) + self.status, self.code, self.message = status, code, message + + +def sniff(head): + if head.startswith(PNG_SIGNATURE): + return "image/png" + if head[:3] == b"\xff\xd8\xff": + return "image/jpeg" + return None + + +def _check_png_stream(fileobj, width, height): + """Целостность потока IDAT без аллокации изображения: распаковка порциями, проверка конца потока и размера данных.""" + fileobj.seek(8) + inflater = zlib.decompressobj() + total, channels, bit_depth, interlaced = 0, 0, 0, 0 + while True: + head = fileobj.read(8) + if len(head) < 8: + raise ValueError("truncated") + length, kind = struct.unpack(">I4s", head) + if length > 0x7FFFFFFF: + raise ValueError("chunk") + if kind == b"IHDR": + data = fileobj.read(length) + bit_depth, color_type, _, _, interlaced = struct.unpack(">BBBBB", data[8:13]) + channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[color_type] + fileobj.read(4) + elif kind == b"IDAT": + remaining = length + while remaining: + piece = fileobj.read(min(remaining, 65536)) + if not piece: + raise ValueError("truncated") + remaining -= len(piece) + pending = piece + while pending: + total += len(inflater.decompress(pending, 1 << 20)) + pending = inflater.unconsumed_tail + fileobj.read(4) + elif kind == b"IEND": + break + else: + fileobj.seek(length + 4, 1) + if not inflater.eof: + raise ValueError("zlib stream incomplete") + if not interlaced: + row = (width * channels * bit_depth + 7) // 8 + if total != height * (row + 1): + raise ValueError("unexpected image data size") + + +def validate(fileobj, declared_type): + """Возвращает (content_type, width, height) либо бросает ImageRejected (415 — не то/повреждено, 413 — много пикселей).""" + fileobj.seek(0) + detected = sniff(fileobj.read(16)) + if detected is None or detected != declared_type: + raise ImageRejected(415, "unsupported_media_type", "Содержимое не соответствует PNG/JPEG или заявленному Content-Type") + fileobj.seek(0) + try: + with Image.open(fileobj) as img: + width, height = img.size + fmt = img.format + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError): + raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено или не поддерживается") from None + if fmt not in ("PNG", "JPEG") or width < 1 or height < 1: + raise ImageRejected(415, "unsupported_media_type", "Неподдерживаемый формат изображения") + if width * height > config.MAX_IMAGE_PIXELS: + raise ImageRejected(413, "payload_too_large", "Слишком много пикселей в изображении") + fileobj.seek(0) + try: + img = Image.open(fileobj) + if fmt == "PNG": + img.verify() + _check_png_stream(fileobj, width, height) + else: + img.draft("RGB", (max(1, width // 8), max(1, height // 8))) + img.load() + img.close() + except Exception: # любая ошибка декодера = повреждённый файл + raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено") from None + return detected, width, height diff --git a/src/app/main.py b/src/app/main.py new file mode 100644 index 0000000..4575a0d --- /dev/null +++ b/src/app/main.py @@ -0,0 +1,464 @@ +"""ASGI-приложение мессенджера (lab2): REST API, PostgreSQL, сессии Bearer, кэш Valkey.""" +import asyncio +import contextlib +from concurrent.futures import ThreadPoolExecutor +import base64 +import binascii +import hashlib +import json +import os +import re +import time +import uuid + +import asyncpg + +from . import config, passwords, telemetry +from .attachments import AttachmentMixin, StreamResponse +from .db import DEPENDENCY_ERRORS, Database +from .broker import Broker +from .outbox import Relay +from .s3 import S3 +from .sessions import SessionCache, Sessions +from .web import (ApiError, Request, bad, dt_to_us, encode_cursor, format_dt, log, new_request_id, page_params, + parse_json_object, parse_uuid, respond, unavailable, us_to_dt, valid_text, USERNAME_RE) + +ROUTES = [ + ("health_live", re.compile(r"/health/live"), {"GET"}), + ("health_ready", re.compile(r"/health/ready"), {"GET"}), + ("users", re.compile(r"/api/v1/users"), {"POST"}), + ("users_me", re.compile(r"/api/v1/users/me"), {"GET"}), + ("create_session", re.compile(r"/api/v1/auth/sessions"), {"POST"}), + ("session_current", re.compile(r"/api/v1/auth/sessions/current"), {"DELETE"}), + ("chats", re.compile(r"/api/v1/chats"), {"GET", "POST"}), + ("chat", re.compile(r"/api/v1/chats/(?P[^/]+)"), {"GET"}), + ("messages", re.compile(r"/api/v1/chats/(?P[^/]+)/messages"), {"GET", "POST"}), + ("upload_attachment", re.compile(r"/api/v1/chats/(?P[^/]+)/attachments"), {"POST"}), + ("get_attachment", re.compile(r"/api/v1/attachments/(?P[^/]+)"), {"GET"}), + ("download_attachment", re.compile(r"/api/v1/attachments/(?P[^/]+)/content"), {"GET"}), +] +ROUTE_LABELS = { + "health_live": "/health/live", "health_ready": "/health/ready", "users": "/api/v1/users", + "users_me": "/api/v1/users/me", "create_session": "/api/v1/auth/sessions", + "session_current": "/api/v1/auth/sessions/current", "chats": "/api/v1/chats", + "chat": "/api/v1/chats/{chat_id}", "messages": "/api/v1/chats/{chat_id}/messages", + "upload_attachment": "/api/v1/chats/{chat_id}/attachments", "get_attachment": "/api/v1/attachments/{attachment_id}", + "download_attachment": "/api/v1/attachments/{attachment_id}/content", +} +TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") +BEARER = 'Bearer realm="messenger"' +BASIC = 'Basic realm="messenger", charset="UTF-8"' +# Идентификатор реплики: имя хоста контейнера (12 hex, == короткий ID контейнера) -> легко сопоставить с docker ps. +INSTANCE_ID = os.environ.get("INSTANCE_ID") or (os.environ.get("HOSTNAME") or uuid.uuid4().hex)[:12] + +MSG_ATTS = ("COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) FROM message_attachments ma " + "WHERE ma.message_id = messages.id), '{}'::uuid[]) AS attachment_ids") +CHAT_SELECT = ("SELECT c.id, c.title, c.created_at, " + "(SELECT array_agg(m2.user_id ORDER BY m2.ord) FROM chat_members m2 WHERE m2.chat_id = c.id) AS member_ids ") + + +def user_json(row): + return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], + "created_at": format_dt(row["created_at"])} + + +def chat_json(row): + return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in row["member_ids"]], + "created_at": format_dt(row["created_at"])} + + +def message_json(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), + "text": row["text"], "created_at": format_dt(row["created_at"]), + "attachment_ids": [str(x) for x in (row["attachment_ids"] or [])]} + + +def unauthorized(challenge=BEARER, message="Требуется действительная сессия"): + return ApiError(401, "unauthorized", message, {"www-authenticate": challenge}) + + +class Messenger(AttachmentMixin): + def __init__(self): + self.db = Database() + self.s3 = S3() + self.broker = Broker() + self.relay = Relay(self.db, self.broker) + self.image_pool = ThreadPoolExecutor(max_workers=config.IMAGE_THREADS, thread_name_prefix="image") + self._init_attachments() + self.cache = SessionCache() + self.sessions = Sessions(self.db, self.cache) + self.inflight = 0 + self.draining = False + # Метрики очереди считает API (любая реплика), а не worker: при остановке worker алерт по возрасту задач продолжает работать. + self.queue_stats = (0, 0.0) + telemetry.gauge("messenger.queue.depth", lambda: [(self.queue_stats[0], {"queue.name": "attachment.process"})], "{job}", "queued + processing") + telemetry.gauge("messenger.queue.oldest_job_age", lambda: [(self.queue_stats[1], {"queue.name": "attachment.process"})], "s", "Возраст старейшего задания") + + # --- аутентификация ------------------------------------------------------ + @staticmethod + def bearer_token(req): + header = req.headers.get("authorization", "") + scheme, _, token = header.partition(" ") + if scheme.lower() != "bearer" or not TOKEN_RE.fullmatch(token.strip()): + raise unauthorized() + return token.strip() + + async def authenticate(self, req): + token = self.bearer_token(req) + user_id = await self.sessions.resolve(token) + if user_id is None: + raise unauthorized() + req.user_id, req.token = user_id, token + return user_id + + # --- обработчики ----------------------------------------------------------- + async def health_live(self, req, params): + return 200, {"status": "ok"} + + async def health_ready(self, req, params): + # Критичная зависимость — PostgreSQL. Valkey — необязательный кэш: его отказ не останавливает API. + if self.draining: + raise unavailable("Сервис останавливается") + try: + await self.db.ping() + except ApiError: + raise + except Exception: + raise unavailable() from None + return 200, {"status": "ok"} + + async def users(self, req, params): + req.require_json_media_type() + data = parse_json_object(await req.body(), {"username", "display_name", "password"}, + {"username", "display_name", "password"}) + username = data["username"] + if not isinstance(username, str) or not USERNAME_RE.fullmatch(username): + raise bad("username должен соответствовать [a-z0-9_]{3,32}") + display_name = valid_text(data["display_name"], "display_name", 100) + password = valid_text(data["password"], "password", 128, allow_empty=True) + if len(password) < 12: + raise bad("password должен содержать от 12 до 128 символов") + pw_hash = await passwords.hash_password(password) + row = await self.db.fetchrow( + "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) " + "ON CONFLICT (username) DO NOTHING RETURNING id, username, display_name, created_at", + uuid.uuid4(), username, display_name, pw_hash) + if row is None: + raise ApiError(409, "conflict", "username уже занят") + return 201, user_json(row) + + async def users_me(self, req, params): + user_id = await self.authenticate(req) + row = await self.db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id) + if row is None: + raise unauthorized() + return 200, user_json(row) + + async def create_session(self, req, params): + header = req.headers.get("authorization", "") + scheme, _, value = header.partition(" ") + if scheme.lower() != "basic" or not value.strip(): + raise unauthorized(BASIC, "Неверные учётные данные") + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + username, sep, password = decoded.partition(":") + if not sep: + raise ValueError + except (binascii.Error, ValueError, UnicodeDecodeError): + raise unauthorized(BASIC, "Неверные учётные данные") from None + row = None + if USERNAME_RE.fullmatch(username): + row = await self.db.fetchrow("SELECT id, username, display_name, created_at, password_hash FROM users " + "WHERE username = $1", username) + ok = await passwords.verify_password(row["password_hash"] if row else None, password) + if not ok: + raise unauthorized(BASIC, "Неверные учётные данные") + token, expires_at = await self.sessions.create(row["id"]) + return 201, {"token": token, "token_type": "Bearer", "expires_at": format_dt(expires_at), "user": user_json(row)} + + async def session_current(self, req, params): + await self.authenticate(req) + await self.sessions.revoke(req.token) + return 204, None + + async def chats(self, req, params): + user_id = await self.authenticate(req) + if req.method == "GET": + limit, after = page_params(req.query, "c", user_id) + if after is None: + rows = await self.db.fetch( + CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 " + "ORDER BY m.chat_created_at, m.chat_id LIMIT $2", user_id, limit + 1) + else: + rows = await self.db.fetch( + CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 " + "AND (m.chat_created_at, m.chat_id) > ($2, $3) ORDER BY m.chat_created_at, m.chat_id LIMIT $4", + user_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1) + more = len(rows) > limit + rows = rows[:limit] + cursor = encode_cursor("c", user_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None + return 200, {"items": [chat_json(r) for r in rows], "next_cursor": cursor} + req.require_json_media_type() + data = parse_json_object(await req.body(), {"title", "member_ids"}, {"title", "member_ids"}) + title = valid_text(data["title"], "title", 100) + members = data["member_ids"] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + raise bad("member_ids должен содержать от 1 до 100 элементов") + ids = [parse_uuid(m, "member_id") for m in members] + if len(set(ids)) != len(ids): + raise bad("member_ids содержит дубликаты") + if user_id not in ids: + raise bad("Создатель должен быть в member_ids") + chat_id = uuid.uuid4() + uuids = [uuid.UUID(i) for i in ids] + try: + async with self.db.conn() as c: + async with c.transaction(): # чат и участники создаются атомарно + # Существование участников проверяет внешний ключ chat_members.user_id: при неизвестном + # участнике вставка падает и откатывается вся транзакция, включая уже созданный чат. + created = await c.fetchval("INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at", + chat_id, title) + await c.execute( + "INSERT INTO chat_members (chat_id, user_id, chat_created_at, ord) " + "SELECT $1, u, $2, o FROM unnest($3::uuid[]) WITH ORDINALITY AS t(u, o)", + chat_id, created, uuids) + except asyncpg.ForeignKeyViolationError: + raise bad("Неизвестный участник") from None + return 201, {"id": str(chat_id), "title": title, "member_ids": ids, "created_at": format_dt(created)} + + async def _require_member(self, user_id, params): + chat_id = parse_uuid(params["chat_id"], "chat_id") + ok = await self.db.fetchval("SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user_id) + if not ok: + raise ApiError(404, "not_found", "Чат не найден") + return chat_id + + async def chat(self, req, params): + user_id = await self.authenticate(req) + chat_id = parse_uuid(params["chat_id"], "chat_id") + row = await self.db.fetchrow( + CHAT_SELECT + "FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 WHERE c.id = $1", + chat_id, user_id) + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + return 200, chat_json(row) + + async def messages(self, req, params): + user_id = await self.authenticate(req) + if req.method == "POST": + row = await self._post_message(req, parse_uuid(params["chat_id"], "chat_id"), user_id) + return 201, message_json(row) + chat_id = await self._require_member(user_id, params) + if req.method == "GET": + limit, after = page_params(req.query, "m", chat_id) + if after is None: + rows = await self.db.fetch( + "SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 " + "ORDER BY created_at, id LIMIT $2", chat_id, limit + 1) + else: + rows = await self.db.fetch( + "SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 " + "AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4", + chat_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1) + more = len(rows) > limit + rows = rows[:limit] + cursor = encode_cursor("m", chat_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None + return 200, {"items": [message_json(r) for r in rows], "next_cursor": cursor} + + async def _post_message(self, req, chat_id, user_id): + # Сначала разбор запроса; ошибка валидации (400/415) возвращается только участнику чата — постороннему всегда 404. + try: + key = req.headers.get("idempotency-key", "") + if not IDEMPOTENCY_KEY_RE.fullmatch(key): + raise bad("Требуется заголовок Idempotency-Key ([A-Za-z0-9._:-]{1,128})") + req.require_json_media_type() + data = parse_json_object(await req.body(), {"text"}, {"text", "attachment_ids"}) + raw_atts = data.get("attachment_ids", []) + if not isinstance(raw_atts, list) or len(raw_atts) > 10: + raise bad("attachment_ids: массив до 10 элементов") + atts = [parse_uuid(a, "attachment_id") for a in raw_atts] + if len(set(atts)) != len(atts): + raise bad("attachment_ids содержит дубликаты") + text = valid_text(data["text"], "text", 4000, allow_empty=bool(atts)) + except ApiError: + await self._require_member(user_id, {"chat_id": chat_id}) + raise + # Нормализованное представление запроса: порядок ключей JSON не важен, отсутствие attachment_ids == []. + request_hash = hashlib.sha256(json.dumps({"text": text, "attachment_ids": atts}, sort_keys=True, + ensure_ascii=False).encode("utf-8")).digest() + row = await self.db.fetchrow("SELECT * FROM send_message($1, $2, $3, $4, $5, $6, $7)", uuid.UUID(chat_id), + uuid.UUID(user_id), key, request_hash, text, float(config.IDEMPOTENCY_TTL_SECONDS), + [uuid.UUID(a) for a in atts], retry=False) + if row["outcome"] == "not_member": + raise ApiError(404, "not_found", "Чат не найден") + if row["outcome"] == "attachment_not_found": + raise ApiError(404, "not_found", "Вложение не найдено") + if row["outcome"] == "attachment_failed": + raise ApiError(409, "conflict", "Вложение не удалось обработать") + if bytes(row["request_hash"]) != request_hash: + raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим телом запроса") + return {"id": row["msg_id"], "chat_id": row["msg_chat"], "sender_id": row["msg_sender"], "text": row["msg_text"], + "created_at": row["msg_created"], "attachment_ids": row["msg_atts"]} + + # --- ASGI ---------------------------------------------------------------------- + async def __call__(self, scope, receive, send): + if scope["type"] == "lifespan": + return await self.lifespan(receive, send) + if scope["type"] != "http": + return + started = time.perf_counter() + request_id = new_request_id() + route_name, status, headers = "unknown", 500, {} + raw_headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]} + carrier = {k: raw_headers[k] for k in ("traceparent", "tracestate") if k in raw_headers} + is_health = scope.get("path", "").startswith("/health") # проверки LB раз в секунду: без спанов и access-log (метрики остаются) + span_cm = (contextlib.nullcontext() if is_health else + telemetry.span("HTTP %s" % scope.get("method"), "server", {"http.request.method": scope.get("method")}, carrier)) + sp = span_cm.__enter__() + try: + req = Request(scope, receive, request_id) + handler, params, route_name = self.route(req.method, req.path) + telemetry.set_attr(sp, "http.route", ROUTE_LABELS.get(route_name, "unmatched")) + req.span = sp + if not route_name.startswith("health") and self.inflight >= config.MAX_INFLIGHT: + raise ApiError(429, "rate_limited", "Слишком много одновременных запросов", + {"retry-after": str(config.RETRY_AFTER_SECONDS)}) + self.inflight += 1 + try: + await asyncio.sleep(0) + if not route_name.startswith("health") and (config.HTTP_FAULT_500 or config.HTTP_FAULT_SLOW): + await self.inject_http_faults() + status, body = await handler(req, params) + finally: + self.inflight -= 1 + except ApiError as exc: + status, headers = exc.status, exc.headers + body = {"error": {"code": exc.code, "message": exc.message, "request_id": request_id}} + except DEPENDENCY_ERRORS as exc: + log("warning", "dependency_error", request_id=request_id, error_type=type(exc).__name__) + status, headers = 503, {"retry-after": str(config.RETRY_AFTER_SECONDS)} + body = {"error": {"code": "unavailable", "message": "Зависимость временно недоступна", "request_id": request_id}} + except Exception as exc: + log("error", "unhandled_exception", request_id=request_id, error_type=type(exc).__name__) + status = 500 + body = {"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}} + route_label = ROUTE_LABELS.get(route_name, "unmatched") + duration = time.perf_counter() - started + telemetry.set_attr(sp, "http.response.status_code", status) + telemetry.set_attr(sp, "http.request_id", request_id) + if sp is not None: + sp.update_name("HTTP %s %s" % (scope.get("method"), route_label)) + if status >= 500: + telemetry.mark_error(sp, "HTTP %d" % status) + telemetry.record_http(scope.get("method"), route_label, status, duration) + if isinstance(body, StreamResponse): + try: + await body.send(send, request_id, INSTANCE_ID) + except Exception as exc: # клиент оборвал загрузку/ошибка S3 посреди потока + log("warning", "stream_aborted", request_id=request_id, error_type=type(exc).__name__) + else: + await respond(send, status, body, request_id, headers, INSTANCE_ID) + if os.environ.get("ACCESS_LOG", "1") != "0" and (not is_health or status >= 500): + log("info", "request", request_id=request_id, method=scope.get("method"), + route=route_label, status=status, duration_ms=round(duration * 1000, 3)) + span_cm.__exit__(None, None, None) + + async def janitor(self): + """Удаляет давно просроченные ключи идемпотентности и сессии (безопасно запускать на каждой реплике).""" + import random + await asyncio.sleep(random.uniform(5, 30)) + while True: + try: + await self.db.execute("DELETE FROM idempotency_keys WHERE ctid IN (SELECT ctid FROM idempotency_keys " + "WHERE created_at < now() - make_interval(secs => $1) LIMIT 5000)", + float(config.IDEMPOTENCY_TTL_SECONDS) + 3600) + await self.db.execute("DELETE FROM sessions WHERE ctid IN (SELECT ctid FROM sessions " + "WHERE expires_at < now() - interval '1 day' LIMIT 5000)") + except Exception as exc: + log("warning", "janitor_failed", error_type=type(exc).__name__) + await asyncio.sleep(3600) + + async def queue_stats_loop(self): + while True: + try: + row = await self.db.fetchrow("SELECT count(*) AS n, coalesce(extract(epoch FROM now() - min(accepted_at)), 0) AS age " + "FROM attachments WHERE status IN ('queued', 'processing')") + self.queue_stats = (row["n"], float(row["age"])) + except Exception: + pass + await asyncio.sleep(2) + + @staticmethod + async def inject_http_faults(): + import random + if config.HTTP_FAULT_SLOW and random.random() < float(config.HTTP_FAULT_SLOW[0]): + await asyncio.sleep(float(config.HTTP_FAULT_SLOW[1])) + if config.HTTP_FAULT_500 and random.random() < float(config.HTTP_FAULT_500[0]): + raise RuntimeError("injected fault") + + def route(self, method, path): + for name, pattern, methods in ROUTES: + m = pattern.fullmatch(path) + if m: + if method not in methods: + raise ApiError(405, "invalid_request", "Метод не поддерживается", {"allow": ", ".join(sorted(methods))}) + return getattr(self, name), m.groupdict(), name + raise ApiError(404, "not_found", "Ресурс не найден") + + async def lifespan(self, receive, send): + while True: + event = await receive() + if event["type"] == "lifespan.startup": + telemetry.setup(config.SERVICE_NAME) + telemetry.process_metrics() + await self.db.start() + await self.cache.start() + await self.s3.start() + self._relay_task = asyncio.create_task(self.relay.run()) + self._queue_task = asyncio.create_task(self.queue_stats_loop()) + self._janitor = asyncio.create_task(self.janitor()) + log("info", "startup", env=config.APP_ENV, instance=INSTANCE_ID, revision=os.environ.get("APP_REVISION", "dev"), cache=bool(config.VALKEY_URL), + session_ttl=config.SESSION_TTL_SECONDS, pool_max=config.DB_POOL_MAX) + await send({"type": "lifespan.startup.complete"}) + elif event["type"] == "lifespan.shutdown": + self.draining = True + self._janitor.cancel() + await self.cache.stop() + telemetry.shutdown() + self.relay.stop() + self._relay_task.cancel() + self._queue_task.cancel() + await self.broker.stop() + await self.s3.stop() + await self.db.stop() + log("info", "shutdown") + await send({"type": "lifespan.shutdown.complete"}) + return + + +async def overloaded_app(scope, receive, send): + """Ответ uvicorn при превышении --limit-concurrency в контрактном формате.""" + if scope["type"] != "http": + return + request_id = new_request_id() + body = {"error": {"code": "unavailable", "message": "Сервер перегружен: слишком много соединений", + "request_id": request_id}} + await respond(send, 503, body, request_id, {"retry-after": str(config.RETRY_AFTER_SECONDS), "connection": "close"}, INSTANCE_ID) + + +def _install_overload_response(): + for name in ("uvicorn.protocols.http.httptools_impl", "uvicorn.protocols.http.h11_impl"): + try: + module = __import__(name, fromlist=["service_unavailable"]) + except ImportError: + continue + module.service_unavailable = overloaded_app + + +def create_app(): + _install_overload_response() + return Messenger() + + +app = create_app() diff --git a/src/app/migrate.py b/src/app/migrate.py new file mode 100644 index 0000000..e979d89 --- /dev/null +++ b/src/app/migrate.py @@ -0,0 +1,64 @@ +"""Применение версионируемых SQL-миграций: python -m app.migrate + +Свойства: + * повторный запуск ничего не делает (таблица schema_migrations); + * одновременный запуск нескольких экземпляров безопасен (advisory lock); + * каждая миграция — одна транзакция вместе с записью о применении; + * ждёт готовности PostgreSQL до MIGRATE_WAIT_SECONDS. +""" +import asyncio +import hashlib +import os +import sys +import time + +import asyncpg + +from . import config +from .web import log + +LOCK_ID = 7352001 + + +async def connect(deadline): + while True: + try: + return await asyncpg.connect(config.DATABASE_URL, timeout=3) + except (OSError, asyncpg.PostgresError, asyncio.TimeoutError) as exc: + if time.monotonic() > deadline: + raise + log("info", "waiting_for_database", error_type=type(exc).__name__) + await asyncio.sleep(1) + + +async def main(): + wait = float(os.environ.get("MIGRATE_WAIT_SECONDS", "60")) + conn = await connect(time.monotonic() + wait) + try: + await conn.execute("SELECT pg_advisory_lock($1)", LOCK_ID) + await conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (" + "version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())") + applied = {r["version"]: r["checksum"] for r in await conn.fetch("SELECT version, checksum FROM schema_migrations")} + directory = os.path.abspath(config.MIGRATIONS_DIR) + files = sorted(f for f in os.listdir(directory) if f.endswith(".sql")) + count = 0 + for name in files: + sql = open(os.path.join(directory, name), encoding="utf-8").read() + checksum = hashlib.sha256(sql.encode("utf-8")).hexdigest() + if name in applied: + if applied[name] != checksum: + log("error", "migration_modified", version=name) + sys.exit(2) + continue + async with conn.transaction(): + await conn.execute(sql) + await conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)", name, checksum) + count += 1 + log("info", "migration_applied", version=name) + log("info", "migrations_done", applied=count, total=len(files)) + finally: + await conn.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/app/outbox.py b/src/app/outbox.py new file mode 100644 index 0000000..433e6b6 --- /dev/null +++ b/src/app/outbox.py @@ -0,0 +1,82 @@ +"""Transactional outbox: relay публикует задания из PostgreSQL в брокер. + +Гарантия: задание, зафиксированное в БД (COMMIT), будет опубликовано даже если процесс упал между COMMIT и publish. +Relay запускается в каждой реплике API и в worker; строки берутся `FOR UPDATE SKIP LOCKED`, поэтому одну строку +публикует один relay. Если процесс упадёт после подтверждения брокера, но до отметки published_at, задание будет +опубликовано повторно — доставка at-least-once, дубликаты безвредны (обработка идемпотентна). +""" +import asyncio +from datetime import timezone + +from . import config, telemetry +from .web import format_dt, log + +JOB_TYPE = "attachment.process.v1" + + +def job_message(row, carrier=None): + msg = {"schema_version": 1, "job_id": str(row["job_id"]), "type": JOB_TYPE, "attachment_id": str(row["attachment_id"]), + "pipeline_version": row["pipeline_version"], "attempt": row["attempt"], + "created_at": format_dt(row["created_at"].astimezone(timezone.utc))} + msg.update(carrier or {}) # traceparent / tracestate (W3C Trace Context) — поля схемы jobs.schema.json + return msg + + +class Relay: + def __init__(self, db, broker): + self.db, self.broker = db, broker + self._kick = asyncio.Event() + self._stop = False + self.last_error = None + + def kick(self): + self._kick.set() + + def stop(self): + self._stop = True + self._kick.set() + + async def relay_once(self): + published = 0 + async with self.db.conn() as c: + async with c.transaction(): + rows = await c.fetch( + "SELECT id, job_id, attachment_id, pipeline_version, attempt, created_at, traceparent, tracestate FROM outbox " + "WHERE published_at IS NULL AND available_at <= now() ORDER BY id LIMIT $1 FOR UPDATE SKIP LOCKED", + config.RELAY_BATCH) + for row in rows: + parent = {k: row[k] for k in ("traceparent", "tracestate") if row[k]} + # PRODUCER-спан — дочерний к спану запроса upload (контекст восстановлен из outbox) + with telemetry.span("attachment.process publish", "producer", + {"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process", + "messaging.operation": "publish", "outbox.attempt": row["attempt"]}, parent): + carrier = telemetry.current_carrier() or parent + await self.broker.publish(job_message(row, carrier), headers=carrier) + await c.execute("UPDATE outbox SET published_at = now() WHERE id = $1", row["id"]) + published += 1 + return published + + def headers_for(self, row): + return {} + + async def run(self): + delay = config.RELAY_INTERVAL + while not self._stop: + try: + n = await self.relay_once() + self.last_error = None + delay = config.RELAY_INTERVAL + if n >= config.RELAY_BATCH: + continue # возможно, есть ещё + except asyncio.CancelledError: + raise + except Exception as exc: + if self.last_error != type(exc).__name__: + log("warning", "outbox_relay_failed", error_type=type(exc).__name__) + self.last_error = type(exc).__name__ + delay = min(delay * 2, 10.0) # брокер/БД недоступны: пауза с ростом до 10 с, задания остаются в outbox + try: + await asyncio.wait_for(self._kick.wait(), delay) + except asyncio.TimeoutError: + pass + self._kick.clear() diff --git a/src/app/passwords.py b/src/app/passwords.py new file mode 100644 index 0000000..650e7b3 --- /dev/null +++ b/src/app/passwords.py @@ -0,0 +1,31 @@ +"""Хеширование паролей: Argon2id (библиотека argon2-cffi), вычисления вне event loop.""" +import asyncio +from concurrent.futures import ThreadPoolExecutor + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError + +from . import config + +_hasher = PasswordHasher(time_cost=config.ARGON2_TIME_COST, memory_cost=config.ARGON2_MEMORY_KIB, + parallelism=config.ARGON2_PARALLELISM) +_pool = ThreadPoolExecutor(max_workers=config.HASH_CONCURRENCY, thread_name_prefix="argon2") +# Для неизвестного username проверяем фиктивный хеш: время ответа не выдаёт существование пользователя. +_DUMMY = _hasher.hash("dummy-password-for-timing-equalisation") + + +async def hash_password(password): + return await asyncio.get_running_loop().run_in_executor(_pool, _hasher.hash, password) + + +def _verify(stored, password): + try: + return _hasher.verify(stored, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +async def verify_password(stored, password): + """stored=None -> проверка фиктивного хеша, результат всегда False.""" + ok = await asyncio.get_running_loop().run_in_executor(_pool, _verify, stored or _DUMMY, password) + return ok and stored is not None diff --git a/src/app/pipeline.py b/src/app/pipeline.py new file mode 100644 index 0000000..a38af25 --- /dev/null +++ b/src/app/pipeline.py @@ -0,0 +1,97 @@ +"""Обработка изображений (pipeline_version=1): thumbnail, crop, optimized. + +Правила (contracts/README.md): + * thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, БЕЗ увеличения, EXIF orientation применяется, alpha — на белый + фон; размеры округляются вниз, минимум 1 px (320x200 -> 256x160); качество 85, метаданные не сохраняются; + * crop: JPEG, центрированный квадрат 128x128 (маленькие исходники допускается увеличивать), качество 85; + * optimized: WebP, вписан в 1280x1280 без увеличения, quality=80 method=4, EXIF/ICC не сохраняются (ориентация применена). +Память: JPEG декодируется через draft (масштабирование при декодировании), число пикселей ограничено MAX_IMAGE_PIXELS. +""" +import hashlib +import io +import math + +from PIL import Image, ImageOps + +from . import config + +Image.MAX_IMAGE_PIXELS = None +PIPELINE_VERSION = 1 +JPEG_QUALITY = 85 +WEBP_QUALITY = 80 + + +class PermanentError(Exception): + """Постоянная ошибка входных данных: повтор не поможет (-> failed, error_code).""" + + def __init__(self, code): + super().__init__(code) + self.code = code + + +def _flatten(img): + """RGBA/LA/P с прозрачностью -> RGB на белом фоне.""" + if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info): + rgba = img.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + return img.convert("RGB") + + +def _fit_size(width, height, box, upscale=False): + scale = min(box / width, box / height) + if not upscale: + scale = min(scale, 1.0) + return max(1, math.floor(width * scale)), max(1, math.floor(height * scale)) + + +def _encode(img, fmt, **params): + buf = io.BytesIO() + img.save(buf, fmt, **params) + data = buf.getvalue() + return data, hashlib.sha256(data).hexdigest() + + +def process(fileobj): + """Возвращает список вариантов: dict(name, content_type, data, sha256, width, height). Бросает PermanentError.""" + try: + fileobj.seek(0) + src = Image.open(fileobj) + fmt = src.format + width, height = src.size + if fmt not in ("PNG", "JPEG"): + raise PermanentError("invalid_image") + if width * height > config.MAX_IMAGE_PIXELS: + raise PermanentError("image_too_large") + if fmt == "JPEG" and width > 2560 and height > 2560: + # экономия памяти: JPEG декодируется сразу в масштабе 1/2 (результат всё равно вписывается в 1280x1280) + src.draft("RGB", (width // 2, height // 2)) + src.load() + img = ImageOps.exif_transpose(src) + except PermanentError: + raise + except MemoryError: + raise + except Exception: + raise PermanentError("invalid_image") from None + base = _flatten(img) + out = [] + # thumbnail + tw, th = _fit_size(base.width, base.height, 256) + thumb = base.resize((tw, th), Image.LANCZOS) if (tw, th) != base.size else base + data, sha = _encode(thumb, "JPEG", quality=JPEG_QUALITY, optimize=True) + out.append({"name": "thumbnail", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": tw, "height": th}) + # crop 128x128 (центрированный квадрат) + side = min(base.width, base.height) + left, top = (base.width - side) // 2, (base.height - side) // 2 + crop = base.crop((left, top, left + side, top + side)).resize((128, 128), Image.LANCZOS) + data, sha = _encode(crop, "JPEG", quality=JPEG_QUALITY, optimize=True) + out.append({"name": "crop", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": 128, "height": 128}) + # optimized WebP (прозрачность сохраняется) + opt_src = img.convert("RGBA") if (img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info)) else base + ow, oh = _fit_size(opt_src.width, opt_src.height, 1280) + opt = opt_src.resize((ow, oh), Image.LANCZOS) if (ow, oh) != opt_src.size else opt_src + data, sha = _encode(opt, "WEBP", quality=WEBP_QUALITY, method=4) + out.append({"name": "optimized", "content_type": "image/webp", "data": data, "sha256": sha, "width": ow, "height": oh}) + return out diff --git a/src/app/reconcile.py b/src/app/reconcile.py new file mode 100644 index 0000000..64038c9 --- /dev/null +++ b/src/app/reconcile.py @@ -0,0 +1,118 @@ +"""Сверка S3 и PostgreSQL (reconcile): python -m app.reconcile [--apply] [--grace СЕКУНДЫ] [--prefix att/] + +Находит и (с --apply) безопасно исправляет расхождения двух хранилищ: + * orphan_object — объект в S3 без строки в attachments (сбой между записью в S3 и COMMIT метаданных, прерванная загрузка); + удаляется ТОЛЬКО если старше grace (по умолчанию ORPHAN_GRACE_SECONDS=900 с): свежий объект может быть + загрузкой, метаданные которой ещё не зафиксированы; + * missing_object — строка attachments (ready) без объекта в S3 (потеря/удаление объекта): строка НЕ удаляется, + статус -> failed, error_code=object_missing (сообщения со ссылкой остаются, скачивание даёт 409); + * size_mismatch — размер объекта не совпадает с size_bytes: статус -> failed, error_code=size_mismatch. +Режим по умолчанию — dry-run (ничего не меняет). Повторный запуск идемпотентен. Используемые (привязанные к сообщению) +вложения и свежие объекты не удаляются никогда: удаляются только объекты, у которых нет строки в БД. +Выход: JSON-отчёт со счётчиками и временем обхода. +""" +import argparse +import asyncio +import json +import time +import uuid +from datetime import datetime, timedelta, timezone + +import asyncpg + +from . import config +from .s3 import S3 + + +def object_owner(key): + """Recognize server-generated prefixes, including effects not committed yet.""" + parts = key.split("/") + if (parts[0] == "att" and len(parts) >= 3) or (parts[0] == "staging" and len(parts) == 2): + try: + return uuid.UUID(parts[1]) + except ValueError: + pass + return None + + +async def reconcile(apply, grace, prefix): + started = time.perf_counter() + s3 = S3() + await s3.start() + conn = await asyncpg.connect(config.DATABASE_URL, timeout=5) + try: + t0 = time.perf_counter() + rows = await conn.fetch("SELECT id, s3_key, staging_key, size_bytes, status, created_at FROM attachments") + variants = await conn.fetch( + "SELECT a.id, v.s3_key, v.size_bytes, a.status, a.created_at " + "FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id") + db_ms = (time.perf_counter() - t0) * 1000 + by_key = {r["s3_key"]: dict(r, check=True) for r in [*rows, *variants]} + # Staging may already be removed after ready, or retained for failed/replay. + by_key.update({r["staging_key"]: dict(r, check=False) for r in rows if r["staging_key"]}) + owners = {r["id"] for r in rows} + t0 = time.perf_counter() + objects = {} + async for key, size, modified in s3.list_keys(prefix): + objects[key] = (size, modified) + list_ms = (time.perf_counter() - t0) * 1000 + now = datetime.now(timezone.utc) + cutoff = now - timedelta(seconds=grace) + report = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace, "db_rows": len(rows), "s3_objects": len(objects), + "orphan_objects": [], "orphans_skipped_fresh": 0, "owned_objects_skipped": 0, + "missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}} + for key, (size, modified) in objects.items(): + row = by_key.get(key) + if row is None: + if object_owner(key) in owners: + # Worker can write S3 variants before committing their metadata. + report["owned_objects_skipped"] += 1 + elif modified < cutoff: + report["orphan_objects"].append(key) + else: + report["orphans_skipped_fresh"] += 1 + elif row["check"] and row["status"] == "ready" and row["size_bytes"] != size: + report["size_mismatch"].append(str(row["id"])) + for key, row in by_key.items(): + if row["check"] and key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff: + report["missing_objects"].append(str(row["id"])) + for field in ("missing_objects", "size_mismatch"): + report[field] = sorted(set(report[field])) + if apply: + for key in report["orphan_objects"]: + # повторная проверка перед удалением: за время обхода строка могла появиться + referenced = await conn.fetchval( + "SELECT EXISTS (SELECT 1 FROM attachments WHERE s3_key = $1 OR staging_key = $1 OR id = $2) " + "OR EXISTS (SELECT 1 FROM attachment_variants WHERE s3_key = $1)", key, object_owner(key)) + if not referenced: + await s3.delete(key) + report["fixed"]["deleted_orphans"] += 1 + for ids, code in ((report["missing_objects"], "object_missing"), (report["size_mismatch"], "size_mismatch")): + for att_id in ids: + n = await conn.execute("UPDATE attachments SET status = 'failed', error_code = $2, updated_at = now() " + "WHERE id = $1 AND status = 'ready'", att_id, code) + report["fixed"]["marked_failed"] += int(n.split()[-1]) + report["timing_ms"] = {"db_query": round(db_ms, 1), "s3_listing": round(list_ms, 1), + "total": round((time.perf_counter() - started) * 1000, 1)} + return report + finally: + await conn.close() + await s3.stop() + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--apply", action="store_true", help="исправлять (по умолчанию dry-run)") + ap.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS) + ap.add_argument("--prefix", default="att/") + ap.add_argument("--max-listed", type=int, default=20, help="сколько идентификаторов печатать в отчёте") + args = ap.parse_args() + report = asyncio.run(reconcile(args.apply, args.grace, args.prefix)) + for k in ("orphan_objects", "missing_objects", "size_mismatch"): + report[k + "_count"] = len(report[k]) + report[k] = report[k][:args.max_listed] + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/app/replay.py b/src/app/replay.py new file mode 100644 index 0000000..d3304c9 --- /dev/null +++ b/src/app/replay.py @@ -0,0 +1,70 @@ +"""Операторский replay и реестр неудавшихся заданий: python -m app.replay [list | replay | replay-all] + + list — реестр failed-вложений: id, error_code, число попыток, возраст, наличие staging; + replay — возвращает ТОЛЬКО failed-вложение в queued: новая попытка (attempt сбрасывается на 1), новый job_id, + запись в job_events (аудит), задание в outbox. Если у вложения уже есть варианты (ready/успешный + эффект), повторного исполнения не будет: вложение не в статусе failed -> replay отклоняется. +Эффекты защищены ключом (attachment_id, pipeline_version, name): даже повторное выполнение не создаёт второй набор вариантов. +""" +import argparse +import asyncio +import json +import sys +import uuid + +import asyncpg + +from . import config + + +async def list_failed(conn): + rows = await conn.fetch( + "SELECT id, error_code, attempt, staging_key IS NOT NULL AS has_staging, updated_at FROM attachments " + "WHERE status = 'failed' ORDER BY updated_at DESC LIMIT 200") + return [{"attachment_id": str(r["id"]), "error_code": r["error_code"], "attempt": r["attempt"], + "staging_present": r["has_staging"], "failed_at": r["updated_at"].isoformat()} for r in rows] + + +async def replay(conn, att_id): + async with conn.transaction(): + row = await conn.fetchrow("SELECT id, status, pipeline_version, staging_key FROM attachments WHERE id = $1 FOR UPDATE", att_id) + if row is None: + return {"attachment_id": str(att_id), "result": "not_found"} + if row["status"] != "failed": + return {"attachment_id": str(att_id), "result": "skipped", "reason": "status=%s (replay только для failed)" % row["status"]} + if not row["staging_key"]: + return {"attachment_id": str(att_id), "result": "skipped", "reason": "staging удалён по retention, исходник недоступен"} + job_id = uuid.uuid4() + await conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, attempt = 0, job_id = $2, lease_owner = NULL, " + "lease_expires_at = NULL, updated_at = now() WHERE id = $1", att_id, job_id) + await conn.execute("INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES ($1, $2, $3, 1)", + job_id, att_id, row["pipeline_version"]) + await conn.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) VALUES ($1, $2, 'operator_replay', 1, 'failed -> queued')", + att_id, job_id) + return {"attachment_id": str(att_id), "result": "requeued", "job_id": str(job_id)} + + +async def main_async(args): + conn = await asyncpg.connect(config.DATABASE_URL, timeout=5) + try: + if args.command == "list": + return await list_failed(conn) + if args.command == "replay": + return await replay(conn, uuid.UUID(args.attachment_id)) + return [await replay(conn, uuid.UUID(f["attachment_id"])) for f in await list_failed(conn)] + finally: + await conn.close() + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="command", required=True) + sub.add_parser("list") + r = sub.add_parser("replay") + r.add_argument("attachment_id") + sub.add_parser("replay-all") + print(json.dumps(asyncio.run(main_async(ap.parse_args())), ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/app/s3.py b/src/app/s3.py new file mode 100644 index 0000000..9d70790 --- /dev/null +++ b/src/app/s3.py @@ -0,0 +1,92 @@ +"""Приватное S3-хранилище (MinIO) через aiobotocore: потоковая загрузка/скачивание, таймауты, перевод сбоев в 503.""" +import asyncio + +from aiobotocore.config import AioConfig +from aiobotocore.session import get_session +from botocore.exceptions import BotoCoreError, ClientError + +from . import config, telemetry +from .web import log, unavailable + +S3_ERRORS = (BotoCoreError, ClientError, asyncio.TimeoutError, OSError) + + +class S3: + def __init__(self): + self._ctx = None + self.client = None + + async def start(self): + if not config.S3_ENDPOINT: + log("error", "s3_endpoint_missing") + return + session = get_session() + self._ctx = session.create_client( + "s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION, + aws_access_key_id=config.S3_ACCESS_KEY, aws_secret_access_key=config.S3_SECRET_KEY, + config=AioConfig(connect_timeout=config.S3_CONNECT_TIMEOUT, read_timeout=config.S3_READ_TIMEOUT, + retries={"max_attempts": 2, "mode": "standard"}, max_pool_connections=config.S3_POOL, + s3={"addressing_style": "path"})) + self.client = await self._ctx.__aenter__() + + async def stop(self): + if self._ctx: + await self._ctx.__aexit__(None, None, None) + + def _check(self): + if self.client is None: + raise unavailable("Хранилище файлов не настроено") + + async def put(self, key, fileobj, content_type, size): + self._check() + try: + with telemetry.child_span("s3.put_object", "client", {"rpc.system": "aws-api", "rpc.method": "PutObject"}): + await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentType=content_type, + ContentLength=size) + except S3_ERRORS as exc: + log("warning", "s3_put_failed", error_type=type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from exc + + async def put_bytes(self, key, data, content_type): + self._check() + with telemetry.child_span("s3.put_object", "client", {"rpc.system": "aws-api", "rpc.method": "PutObject"}): + await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=data, ContentType=content_type) + + async def open_stream(self, key): + """Возвращает (stream, content_length). Вызывающий обязан закрыть stream.""" + self._check() + try: + with telemetry.child_span("s3.get_object", "client", {"rpc.system": "aws-api", "rpc.method": "GetObject"}): + resp = await self.client.get_object(Bucket=config.S3_BUCKET, Key=key) + except S3_ERRORS as exc: + log("warning", "s3_get_failed", error_type=type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from exc + return resp["Body"], resp["ContentLength"] + + async def delete(self, key): + self._check() + await self.client.delete_object(Bucket=config.S3_BUCKET, Key=key) + + async def head(self, key): + """Метаданные объекта или None, если его нет.""" + self._check() + try: + return await self.client.head_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"): + return None + raise unavailable("Хранилище файлов недоступно") from exc + except S3_ERRORS as exc: + raise unavailable("Хранилище файлов недоступно") from exc + + async def ping(self): + self._check() + await asyncio.wait_for(self.client.head_bucket(Bucket=config.S3_BUCKET), 2.0) + + async def list_keys(self, prefix): + """Асинхронный итератор (key, size, last_modified) по префиксу — для сверки (reconcile).""" + self._check() + paginator = self.client.get_paginator("list_objects_v2") + async for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix): + for obj in page.get("Contents", []): + yield obj["Key"], obj["Size"], obj["LastModified"] diff --git a/src/app/s3_bootstrap.py b/src/app/s3_bootstrap.py new file mode 100644 index 0000000..6004a85 --- /dev/null +++ b/src/app/s3_bootstrap.py @@ -0,0 +1,85 @@ +"""Идемпотентный bootstrap S3: python -m app.s3_bootstrap (one-shot сервис s3-init при каждом `docker compose up`). + + 1) bucket создаётся администратором, если его ещё нет (повтор ничего не меняет); + 2) проверяется, что анонимный запрос к bucket отклоняется (private); + 3) проверяется, что отдельный ключ API (права только на этот bucket) может писать/читать/удалять и НЕ может создавать bucket. +Ключи читаются из файлов секретов; в вывод не попадают. Код выхода != 0, если любая проверка не пройдена. +""" +import asyncio +import os +import sys +import time +import urllib.error +import urllib.request + +from aiobotocore.config import AioConfig +from aiobotocore.session import get_session +from botocore.exceptions import ClientError + +from . import config + + +def secret(name): + path = os.environ.get(name + "_FILE") + with open(path, encoding="utf-8") as fh: + return fh.read().strip() + + +def client(session, access, secret_key): + return session.create_client("s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION, aws_access_key_id=access, + aws_secret_access_key=secret_key, + config=AioConfig(connect_timeout=3, read_timeout=10, retries={"max_attempts": 1}, + s3={"addressing_style": "path"})) + + +async def main(): + bucket = config.S3_BUCKET + session = get_session() + deadline = time.time() + 60 + async with client(session, secret("S3_ADMIN_ACCESS_KEY"), secret("S3_ADMIN_SECRET_KEY")) as admin: + while True: + try: + await admin.list_buckets() + break + except Exception as exc: + if time.time() > deadline: + print("S3 недоступен:", type(exc).__name__) + return 1 + await asyncio.sleep(1) + try: + await admin.head_bucket(Bucket=bucket) + print("bucket %s уже существует" % bucket) + except ClientError: + await admin.create_bucket(Bucket=bucket) + print("bucket %s создан" % bucket) + # приватность: запрос без подписи отклоняется + try: + urllib.request.urlopen("%s/%s/" % (config.S3_ENDPOINT, bucket), timeout=5) + print("ОШИБКА: bucket доступен анонимно") + return 2 + except urllib.error.HTTPError as exc: + if exc.code not in (401, 403): + print("ОШИБКА: анонимный доступ вернул %d, ожидался 401/403" % exc.code) + return 2 + print("анонимный доступ запрещён (HTTP %d)" % exc.code) + # ключ API: рабочие права на bucket, нет прав администратора + async with client(session, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) as api: + key = "bootstrap-check/%d" % int(time.time()) + await api.put_object(Bucket=bucket, Key=key, Body=b"ok") + got = await (await api.get_object(Bucket=bucket, Key=key))["Body"].read() + await api.delete_object(Bucket=bucket, Key=key) + if got != b"ok": + print("ОШИБКА: чтение/запись ключом API") + return 3 + try: + await api.create_bucket(Bucket="not-allowed-" + str(int(time.time()))) + print("ОШИБКА: ключ API смог создать bucket (избыточные права)") + return 4 + except ClientError as exc: + print("ключ API ограничен одним bucket (create_bucket отклонён: %s)" % exc.response["Error"]["Code"]) + print("bootstrap завершён") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/src/app/s3_recovery.py b/src/app/s3_recovery.py new file mode 100644 index 0000000..50191d9 --- /dev/null +++ b/src/app/s3_recovery.py @@ -0,0 +1,77 @@ +"""Read-only readiness probe after restart: ranged GET of recent ready originals. + +An open S3 socket / HEAD bucket does not prove that SeaweedFS volume locations +have been registered. Read one byte from up to 16 recent ready originals. +This is a bounded recovery hook, not an audit of every historical object. +""" +import asyncio +import sys + +import asyncpg + +from . import config +from .s3 import S3 + + +async def probe(s3): + count = 0 + batch = [] + + async def read(key, size): + params = {"Bucket": config.S3_BUCKET, "Key": key} + if size: + params["Range"] = "bytes=0-0" + response = await s3.client.get_object(**params) + async with response["Body"] as stream: + data = await stream.read() + if len(data) != min(size, 1): + raise ValueError("unexpected ranged read size") + + async def flush(): + results = await asyncio.gather(*(read(key, size) for key, size in batch), return_exceptions=True) + for result in results: + if isinstance(result, BaseException): + raise result + + await s3.ping() + conn = await asyncpg.connect(config.DATABASE_URL, timeout=5) + try: + rows = await conn.fetch("SELECT s3_key, size_bytes FROM attachments " + "WHERE status = 'ready' ORDER BY accepted_at DESC LIMIT 16") + finally: + await conn.close() + for row in rows: + key, size = row["s3_key"], row["size_bytes"] + batch.append((key, size)) + count += 1 + if len(batch) == 8: + await flush() + batch.clear() + if batch: + await flush() + return count + + +async def main(): + s3 = S3() + await s3.start() + try: + async with asyncio.timeout(45): + while True: + try: + count = await probe(s3) + print(f"PASS: S3 recovery, {count} recent ready originals readable") + return 0 + except Exception as exc: + # Do not print SDK requests/credentials or object contents. + print(f"Waiting for S3 data volumes: {type(exc).__name__}", flush=True) + await asyncio.sleep(1) + except TimeoutError: + print("FAIL: S3 data recovery exceeded 45 seconds") + return 1 + finally: + await s3.stop() + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/src/app/serve.py b/src/app/serve.py new file mode 100644 index 0000000..9980988 --- /dev/null +++ b/src/app/serve.py @@ -0,0 +1,40 @@ +"""Точка входа: uvicorn с управляемым завершением (drain). + +SIGTERM -> readiness сразу 503 -> DRAIN_SECONDS продолжаем обслуживать запросы, пока балансировщик +исключает реплику -> uvicorn прекращает приём и дожидается текущих запросов (timeout_graceful_shutdown). +Так плановая остановка/rolling update не даёт ошибок клиентам. +""" +import signal +import threading + +import uvicorn + +from . import config +from .main import app +from .web import log + + +class DrainingServer(uvicorn.Server): + _draining = False + + def handle_exit(self, sig, frame): + if sig == signal.SIGTERM and not self._draining and config.DRAIN_SECONDS > 0: + self._draining = True + app.draining = True + log("info", "draining", seconds=config.DRAIN_SECONDS) + timer = threading.Timer(config.DRAIN_SECONDS, super().handle_exit, args=(sig, frame)) + timer.daemon = True + timer.start() + return + super().handle_exit(sig, frame) + + +def main(): + cfg = uvicorn.Config(app, host=config.APP_HOST, port=config.APP_PORT, workers=1, access_log=False, + log_level="warning", timeout_graceful_shutdown=config.GRACEFUL_SHUTDOWN_SECONDS, + timeout_keep_alive=config.KEEPALIVE_SECONDS, limit_concurrency=config.LIMIT_CONCURRENCY) + DrainingServer(cfg).run() + + +if __name__ == "__main__": + main() diff --git a/src/app/sessions.py b/src/app/sessions.py new file mode 100644 index 0000000..b07117c --- /dev/null +++ b/src/app/sessions.py @@ -0,0 +1,131 @@ +"""Сессии: PostgreSQL — источник истины; Valkey — кэш с TTL (read-through). + +Политики: + * токен — 256 бит из secrets; в БД и Valkey хранится только SHA-256 токена; + * expiry: expires_at в БД (по часам БД). В кэше TTL = min(остаток жизни, SESSION_CACHE_TTL_SECONDS); + * инвалидация: logout помечает revoked_at в БД и удаляет ключ в Valkey; + * Valkey недоступен -> кэш обходится (чтение из БД), анонимный запрос никогда не считается доверенным; + circuit breaker не даёт каждому запросу ждать таймаут; + * окно устаревания: если Valkey был недоступен во время logout, ключ может прожить до SESSION_CACHE_TTL_SECONDS. +""" +import hashlib +import secrets +import time + +from . import config +from .web import log + +try: + import redis.asyncio as aioredis +except ImportError: # кэш опционален + aioredis = None + + +def token_hash(token): + return hashlib.sha256(token.encode("ascii")).digest() + + +class SessionCache: + def __init__(self): + self.client = None + self.down_until = 0.0 + self.stats = {"hit": 0, "miss": 0, "error": 0} + + async def start(self): + if config.VALKEY_URL and aioredis: + self.client = aioredis.Redis.from_url(config.VALKEY_URL, socket_timeout=config.VALKEY_TIMEOUT, + socket_connect_timeout=config.VALKEY_TIMEOUT) + + async def stop(self): + if self.client: + await self.client.aclose() + + def _usable(self): + return self.client is not None and time.monotonic() >= self.down_until + + def _fail(self, exc): + self.stats["error"] += 1 + if time.monotonic() >= self.down_until: + log("warning", "session_cache_unavailable", error_type=type(exc).__name__) + self.down_until = time.monotonic() + config.VALKEY_BREAKER_SECONDS + + async def get(self, key): + if not self._usable(): + return None + try: + value = await self.client.get(b"s:" + key) + except Exception as exc: + self._fail(exc) + return None + self.stats["hit" if value is not None else "miss"] += 1 + return value + + async def put(self, key, value, ttl): + if not self._usable() or ttl <= 0: + return + try: + await self.client.set(b"s:" + key, value, ex=max(1, int(ttl))) + except Exception as exc: + self._fail(exc) + + async def delete(self, key): + if not self._usable(): + return + try: + await self.client.delete(b"s:" + key) + except Exception as exc: + self._fail(exc) + + async def ping(self): + if self.client is None: + return None + try: + return bool(await self.client.ping()) + except Exception: + return False + + +class Sessions: + def __init__(self, db, cache): + self.db, self.cache = db, cache + + @staticmethod + def _cache_value(user_id, expires_at): + return ("%s|%d" % (user_id, int(expires_at.timestamp() * 1000))).encode() + + async def create(self, user_id): + token = secrets.token_urlsafe(32) # 256 бит энтропии + digest = token_hash(token) + row = await self.db.fetchrow( + "INSERT INTO sessions (token_hash, user_id, expires_at) " + "VALUES ($1, $2, now() + make_interval(secs => $3)) RETURNING expires_at", + digest, user_id, float(config.SESSION_TTL_SECONDS)) + remaining = row["expires_at"].timestamp() - time.time() + await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]), + min(remaining, config.SESSION_CACHE_TTL_SECONDS)) + return token, row["expires_at"] + + async def resolve(self, token): + """token -> user_id (str) либо None, если сессии нет/истекла/отозвана.""" + digest = token_hash(token) + cached = await self.cache.get(digest) + if cached is not None: + user_id, _, expires_ms = cached.decode().partition("|") + if int(expires_ms) > time.time() * 1000: + return user_id + await self.cache.delete(digest) + row = await self.db.fetchrow( + "SELECT user_id, expires_at FROM sessions " + "WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()", digest) + if row is None: + return None + user_id = str(row["user_id"]) + remaining = row["expires_at"].timestamp() - time.time() + await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]), + min(remaining, config.SESSION_CACHE_TTL_SECONDS)) + return user_id + + async def revoke(self, token): + digest = token_hash(token) + await self.db.execute("UPDATE sessions SET revoked_at = now() WHERE token_hash = $1 AND revoked_at IS NULL", digest) + await self.cache.delete(digest) diff --git a/src/app/telemetry.py b/src/app/telemetry.py new file mode 100644 index 0000000..71ef9aa --- /dev/null +++ b/src/app/telemetry.py @@ -0,0 +1,229 @@ +"""Наблюдаемость (lab6): OpenTelemetry SDK — трассы, метрики, логи по OTLP/HTTP в Collector. + +Выключается переменной OTEL_ENABLED=0 (нужно для эксперимента «цена наблюдаемости»): тогда все функции — no-op и +накладных расходов на экспорт нет. Секреты, тела сообщений, токены и изображения в телеметрию не попадают: +в атрибутах только шаблон маршрута, метод, статус, тип задания и исход; ID объектов — только в логах/спанах +(не в labels метрик). + +Соответствие сигналов contracts/observability.md и имён метрик (после Prometheus-экспортёра Collector): + http.server.request.duration (s) -> http_server_request_duration_seconds_{bucket,sum,count} (HTTP requests = _count) + messenger.jobs.processed -> messenger_jobs_processed_total{job_type,outcome} + messenger.job.duration (s) -> messenger_job_duration_seconds_* + messenger.job.time_to_ready (s) -> messenger_job_time_to_ready_seconds_* (от приёма до ready) + messenger.queue.depth / .oldest_age -> messenger_queue_depth{queue_name}, messenger_queue_oldest_job_age_seconds + db.client.connections.usage/waiting -> db_client_connections_usage{state}, db_client_connections_waiting + messenger.worker.busy / .capacity -> messenger_worker_busy, messenger_worker_capacity + process.memory.rss / process.cpu.time -> process_memory_rss_bytes, process_cpu_time_seconds_total +""" +import contextlib +import json +import logging +import os +import time + +from . import config + +ENABLED = config.OTEL_ENABLED +_state = {"tracer": None, "meter": None, "logger": None, "propagator": None, "inst": {}} +HTTP_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0] +JOB_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60] +TTR_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300] + + +def setup(service_name): + """Инициализирует провайдеры. Вызывается один раз при старте процесса (API или worker).""" + if not ENABLED: + return + from opentelemetry import metrics, trace + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.propagate import get_global_textmap + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased + + endpoint = config.OTEL_ENDPOINT.rstrip("/") + resource = Resource.create({"service.name": service_name, "service.version": config.SERVICE_VERSION, + "service.instance.id": config.INSTANCE_ID, "deployment.environment.name": "lab"}) + # --- трассы: head-sampling по доле (ParentBased: решение родителя, в т.ч. из traceparent брокера, сохраняется) + tp = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(config.OTEL_TRACE_RATIO))) + tp.add_span_processor(BatchSpanProcessor( + OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=3), + max_queue_size=config.OTEL_QUEUE_SIZE, max_export_batch_size=config.OTEL_BATCH_SIZE, schedule_delay_millis=2000, export_timeout_millis=3000)) + trace.set_tracer_provider(tp) + # --- метрики (histogram с явными границами, секунды) + views = [View(instrument_name="http.server.request.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)), + View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + View(instrument_name="messenger.job.time_to_ready", aggregation=ExplicitBucketHistogramAggregation(TTR_BUCKETS))] + reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=3), + export_interval_millis=config.OTEL_METRIC_INTERVAL_MS, export_timeout_millis=3000) + mp = MeterProvider(resource=resource, metric_readers=[reader], views=views) + metrics.set_meter_provider(mp) + # --- логи: стандартный logging -> OTLP (trace_id/span_id добавляются автоматически из активного спана) + lp = LoggerProvider(resource=resource) + lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=3), + max_queue_size=config.OTEL_QUEUE_SIZE, schedule_delay_millis=2000)) + logger = logging.getLogger("messenger") + logger.setLevel(logging.INFO) + logger.propagate = False + logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=lp)) + _state.update(tracer=trace.get_tracer("messenger"), meter=metrics.get_meter("messenger"), logger=logger, + propagator=get_global_textmap(), providers=(tp, mp, lp)) + m = _state["meter"] + _state["inst"] = { + "http": m.create_histogram("http.server.request.duration", unit="s", description="Длительность HTTP-запроса"), + "jobs": m.create_counter("messenger.jobs.processed", description="Обработанные задания по исходам"), + "job_dur": m.create_histogram("messenger.job.duration", unit="s", description="Длительность обработки задания"), + "ttr": m.create_histogram("messenger.job.time_to_ready", unit="s", description="От приёма загрузки до ready"), + } + for args in list(_pending_gauges): + (observable_counter if len(args) == 5 else gauge)(*args[:4]) + _pending_gauges.clear() + + +def shutdown(): + for p in _state.get("providers", ()): + try: + p.shutdown() + except Exception: + pass + + +# --- трассы ---------------------------------------------------------------------------------- +@contextlib.contextmanager +def span(name, kind="internal", attributes=None, carrier=None): + """Контекстный менеджер спана. carrier — dict с traceparent/tracestate для продолжения чужого trace.""" + if not ENABLED or _state["tracer"] is None: + yield None + return + from opentelemetry import trace + from opentelemetry.trace import SpanKind, Status, StatusCode + ctx = _state["propagator"].extract(carrier) if carrier else None + kinds = {"server": SpanKind.SERVER, "client": SpanKind.CLIENT, "producer": SpanKind.PRODUCER, + "consumer": SpanKind.CONSUMER, "internal": SpanKind.INTERNAL} + with _state["tracer"].start_as_current_span(name, context=ctx, kind=kinds[kind], attributes=attributes or {}, + record_exception=False, set_status_on_exception=False) as sp: + try: + yield sp + except Exception as exc: + sp.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + sp.set_attribute("error.type", type(exc).__name__) + raise + + +def child_span(name, kind="internal", attributes=None): + """Спан создаётся только внутри активного спана (запрос/задание); фоновые опросы БД не порождают корневых трасс.""" + if not ENABLED or _state["tracer"] is None: + return contextlib.nullcontext() + from opentelemetry import trace + if not trace.get_current_span().get_span_context().is_valid: + return contextlib.nullcontext() + return span(name, kind, attributes) + + +def set_attr(sp, key, value): + if sp is not None: + sp.set_attribute(key, value) + + +def mark_error(sp, description): + if sp is not None: + from opentelemetry.trace import Status, StatusCode + sp.set_status(Status(StatusCode.ERROR, description)) + + +def current_carrier(): + """traceparent/tracestate текущего спана (для передачи через outbox/брокер).""" + if not ENABLED or _state["propagator"] is None: + return {} + carrier = {} + _state["propagator"].inject(carrier) + return carrier + + +def trace_ids(): + if not ENABLED: + return None, None + from opentelemetry import trace + ctx = trace.get_current_span().get_span_context() + if not ctx.is_valid: + return None, None + return "%032x" % ctx.trace_id, "%016x" % ctx.span_id + + +# --- метрики ---------------------------------------------------------------------------------- +def record_http(method, route, status, seconds): + if ENABLED and _state["inst"]: + _state["inst"]["http"].record(seconds, {"http.request.method": method, "http.route": route, + "http.response.status_code": status}) + + +def record_job(job_type, outcome, seconds, time_to_ready=None): + if ENABLED and _state["inst"]: + attrs = {"job.type": job_type, "outcome": outcome} + _state["inst"]["jobs"].add(1, attrs) + _state["inst"]["job_dur"].record(seconds, attrs) + if time_to_ready is not None: + _state["inst"]["ttr"].record(time_to_ready, {"job.type": job_type}) + + +_pending_gauges = [] + + +def gauge(name, callback, unit="1", description=""): + """Observable gauge: callback() -> [(value, {attrs})]. Регистрация откладывается до setup(), если провайдер ещё не создан.""" + if not ENABLED: + return + if _state["meter"] is None: + _pending_gauges.append((name, callback, unit, description)) + return + from opentelemetry.metrics import Observation + + def cb(_options): + try: + return [Observation(v, a) for v, a in callback()] + except Exception: + return [] + _state["meter"].create_observable_gauge(name, callbacks=[cb], unit=unit, description=description) + + +def observable_counter(name, callback, unit="1", description=""): + """Монотонный счётчик, значение которого читается при экспорте (например, накопленное CPU-время).""" + if not ENABLED or _state["meter"] is None: + _pending_gauges.append((name, callback, unit, description, "counter")) + return + from opentelemetry.metrics import Observation + + def cb(_options): + try: + return [Observation(v, a) for v, a in callback()] + except Exception: + return [] + _state["meter"].create_observable_counter(name, callbacks=[cb], unit=unit, description=description) + + +def process_metrics(): + def rss(): + try: + with open("/proc/self/statm") as fh: + return [(int(fh.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"), {})] + except (OSError, ValueError): + return [] + gauge("process.memory.rss", rss, "By", "Резидентная память процесса") + observable_counter("process.cpu.time", lambda: [(time.process_time(), {})], "s", "Накопленное CPU-время процесса") + + +# --- логи --------------------------------------------------------------------------------------- +def emit_log(level, record): + """Отправляет запись в OTLP (если включено). record — dict без секретов.""" + if not ENABLED or _state["logger"] is None: + return + lvl = {"info": logging.INFO, "warning": logging.WARNING, "error": logging.ERROR}.get(level, logging.INFO) + _state["logger"].log(lvl, record.get("msg", ""), extra={k: v for k, v in record.items() if k not in ("msg", "level", "ts")}) diff --git a/src/app/web.py b/src/app/web.py new file mode 100644 index 0000000..cfd75a7 --- /dev/null +++ b/src/app/web.py @@ -0,0 +1,192 @@ +"""Общий HTTP-слой: ошибки, запрос, валидация, курсоры, ответы, журнал.""" +import base64 +import json +import re +import sys +import time +import uuid +from datetime import datetime, timedelta, timezone +from urllib.parse import parse_qsl + +from . import config, telemetry + +UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +LIMIT_RE = re.compile(r"[0-9]{1,9}") +EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) + + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + super().__init__(message) + self.status, self.code, self.message, self.headers = status, code, message, headers or {} + + +def bad(message): + return ApiError(400, "invalid_request", message) + + +def unavailable(message="Зависимость временно недоступна"): + return ApiError(503, "unavailable", message, {"retry-after": str(config.RETRY_AFTER_SECONDS)}) + + +def log(level, msg, **fields): + """Структурный лог: JSON в stdout + OTLP (если включено). trace_id/span_id — из активного спана; тела и токены не логируются.""" + record = {"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "level": level, "service": config.SERVICE_NAME, "msg": msg} + trace_id, span_id = telemetry.trace_ids() + if trace_id: + record["trace_id"], record["span_id"] = trace_id, span_id + record.update(fields) + sys.stdout.write(json.dumps(record, ensure_ascii=False) + "\n") + sys.stdout.flush() + telemetry.emit_log(level, record) + + +# --- время ------------------------------------------------------------------- +def format_dt(value): + value = value.astimezone(timezone.utc) + return value.strftime("%Y-%m-%dT%H:%M:%S.") + "%06dZ" % value.microsecond + + +def dt_to_us(value): + return (value - EPOCH) // timedelta(microseconds=1) + + +def us_to_dt(us): + return EPOCH + timedelta(microseconds=us) + + +# --- валидация --------------------------------------------------------------- +def parse_uuid(value, what="id"): + if not isinstance(value, str) or not UUID_RE.fullmatch(value): + raise bad("Некорректный %s" % what) + return value.lower() + + +def valid_text(value, field, maximum, allow_empty=False): + if not isinstance(value, str): + raise bad("Поле %s должно быть строкой" % field) + try: + value.encode("utf-8") + except UnicodeEncodeError: + raise bad("Поле %s содержит некорректные символы" % field) from None + if "\x00" in value: + raise bad("Поле %s содержит NUL" % field) + if len(value) > maximum: + raise bad("Поле %s слишком длинное" % field) + if not allow_empty and not value.strip(): + raise bad("Поле %s не может быть пустым" % field) + return value + + +def _reject_constant(name): + raise ValueError(name) + + +def parse_json_object(raw, required, allowed): + try: + data = json.loads(raw.decode("utf-8"), parse_constant=_reject_constant) + except (UnicodeDecodeError, ValueError, RecursionError): + raise bad("Тело запроса — некорректный JSON") from None + if not isinstance(data, dict): + raise bad("Ожидается JSON-объект") + if set(data) - allowed: + raise bad("Лишние поля в запросе") + if any(key not in data for key in required): + raise bad("Не хватает обязательных полей") + return data + + +def encode_cursor(kind, resource, key): + payload = json.dumps({"k": kind, "r": resource, "t": key[0], "i": key[1]}, separators=(",", ":")) + return base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=") + + +def decode_cursor(text, kind, resource): + try: + data = json.loads(base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)).decode()) + if (not isinstance(data, dict) or data.get("k") != kind or data.get("r") != resource + or not isinstance(data.get("t"), int) or isinstance(data.get("t"), bool) + or not isinstance(data.get("i"), str) or not UUID_RE.fullmatch(data["i"]) + or not -62135596800000000 < data["t"] < 253402300799000000): + raise ValueError + return data["t"], data["i"] + except (ValueError, UnicodeError): + raise bad("Некорректный cursor") from None + + +def page_params(query, kind, resource): + limit, cursor = 50, None + if "limit" in query: + if len(query["limit"]) != 1 or not LIMIT_RE.fullmatch(query["limit"][0]): + raise bad("limit должен быть целым числом 1–100") + limit = int(query["limit"][0]) + if not 1 <= limit <= 100: + raise bad("limit должен быть целым числом 1–100") + if "cursor" in query: + if len(query["cursor"]) != 1 or not query["cursor"][0]: + raise bad("Некорректный cursor") + cursor = decode_cursor(query["cursor"][0], kind, resource) + return limit, cursor + + +def parse_query(raw): + result = {} + if raw: + for key, value in parse_qsl(raw.decode("latin-1"), keep_blank_values=True): + result.setdefault(key, []).append(value) + return result + + +class Request: + def __init__(self, scope, receive, request_id): + self.method = scope["method"] + self.path = scope["path"] + self.query = parse_query(scope.get("query_string", b"")) + self.headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]} + self.receive = receive + self.request_id = request_id + self.user_id = None + + async def body(self): + declared = self.headers.get("content-length") + if declared is not None: + if not declared.isdigit(): + raise bad("Некорректный Content-Length") + if int(declared) > config.MAX_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"}) + chunks, size = [], 0 + while True: + event = await self.receive() + if event["type"] == "http.disconnect": + raise bad("Соединение закрыто клиентом") + chunk = event.get("body", b"") + size += len(chunk) + if size > config.MAX_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"}) + chunks.append(chunk) + if not event.get("more_body"): + return b"".join(chunks) + + def require_json_media_type(self): + media = self.headers.get("content-type", "").split(";")[0].strip().lower() + if media != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json") + + +async def respond(send, status, body, request_id, extra=None, instance_id=None): + payload = b"" if status == 204 else json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + headers = [(b"x-request-id", request_id.encode())] + if status != 204: + headers += [(b"content-type", b"application/json"), (b"content-length", str(len(payload)).encode())] + else: + headers.append((b"content-length", b"0")) + if instance_id: + headers.append((b"x-instance-id", instance_id.encode())) + headers += [(k.encode(), v.encode()) for k, v in (extra or {}).items()] + await send({"type": "http.response.start", "status": status, "headers": headers}) + await send({"type": "http.response.body", "body": payload}) + + +def new_request_id(): + return uuid.uuid4().hex diff --git a/src/app/worker.py b/src/app/worker.py new file mode 100644 index 0000000..ccad626 --- /dev/null +++ b/src/app/worker.py @@ -0,0 +1,345 @@ +"""Worker вложений: python -m app.worker + +Конвейер задания (jobs.schema.json, состояния — contracts/attachment-states.md): + claim (queued -> processing, lease) -> чтение staging -> варианты (Pillow, в потоках) -> запись в конечный S3-prefix + -> ОДНА транзакция: варианты + ready -> ack -> очистка staging. +Ack отправляется только после устойчивого эффекта (COMMIT результата). Повторная доставка уже готовой задачи — no-op с ack. +Временные ошибки: возврат в queued с backoff (экспоненциальный + jitter) через outbox; после MAX_ATTEMPTS — failed/retry_exhausted. +Постоянные ошибки (невалидное изображение): failed с error_code сразу. Зависшие processing возвращает sweeper по истечении lease. +""" +import asyncio +import json +import os +import random +import signal +import tempfile +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +import aio_pika + +from . import config, pipeline, telemetry +from .broker import QUEUE, Broker +from .db import Database +from .outbox import Relay +from .s3 import S3 +from .web import log + +WORKER_ID = "%s-%s" % ((os.environ.get("HOSTNAME") or "worker")[:12], uuid.uuid4().hex[:6]) + + +def backoff_seconds(attempt): + """Экспоненциальная задержка с полным jitter: случайное значение в [0, min(cap, base * 2^(attempt-1))].""" + ceiling = min(config.RETRY_MAX_DELAY, config.RETRY_BASE_DELAY * (2 ** max(0, attempt - 1))) + return random.uniform(ceiling * 0.5, ceiling) + + +class Worker: + def __init__(self): + self.db, self.s3, self.broker = Database(), S3(), Broker() + self.relay = Relay(self.db, self.broker) + self.pool = ThreadPoolExecutor(max_workers=config.WORKER_CONCURRENCY, thread_name_prefix="img") + self.busy = 0 + self.stopping = False + self.tasks = [] + telemetry.gauge("messenger.worker.busy", lambda: [(self.busy, {})], "{job}", "Задания в обработке") + telemetry.gauge("messenger.worker.capacity", lambda: [(config.WORKER_CONCURRENCY, {})], "{job}", "Ёмкость worker") + + # --- claim / finalize --------------------------------------------------------- + async def claim(self, job): + return await self.db.fetchrow( + "UPDATE attachments SET status = 'processing', lease_owner = $2, processing_started_at = now(), " + "lease_expires_at = now() + make_interval(secs => $3), attempt = $4, job_id = $5, updated_at = now() " + "WHERE id = $1 AND status = 'queued' AND pipeline_version <= $6 " + "RETURNING id, chat_id, staging_key, s3_key, content_type, size_bytes, attempt", + uuid.UUID(job["attachment_id"]), WORKER_ID, float(config.LEASE_SECONDS), job["attempt"], + uuid.UUID(job["job_id"]), job["pipeline_version"], retry=False) + + async def fail(self, att_id, job, code, event="failed"): + await self.db.execute( + "WITH u AS (UPDATE attachments SET status = 'failed', error_code = $2, lease_owner = NULL, lease_expires_at = NULL, " + "updated_at = now() WHERE id = $1 AND status = 'processing' AND lease_owner = $3 RETURNING id) " + "INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $4, $5, $6, $2 FROM u", + att_id, code, WORKER_ID, uuid.UUID(job["job_id"]), event, job["attempt"]) + + async def reschedule(self, att_id, job, reason): + """Временная ошибка: processing -> queued, новая запись outbox с отложенной публикацией (backoff + jitter).""" + nxt = job["attempt"] + 1 + if nxt > config.MAX_ATTEMPTS: + log("warning", "job_retry_exhausted", attachment_id=str(att_id), attempts=job["attempt"]) + await self.fail(att_id, job, "retry_exhausted", "retry_exhausted") + return + delay = backoff_seconds(job["attempt"]) + await self.db.execute( + "WITH u AS (UPDATE attachments SET status = 'queued', lease_owner = NULL, lease_expires_at = NULL, updated_at = now() " + "WHERE id = $1 AND status = 'processing' AND lease_owner = $2 RETURNING id, pipeline_version), " + "o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) " + "SELECT $3, id, pipeline_version, $4, now() + make_interval(secs => $5), $7, $8 FROM u RETURNING 1) " + "INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $3, 'retry_scheduled', $4, $6 FROM u", + att_id, WORKER_ID, uuid.UUID(job["job_id"]), nxt, delay, reason, job.get("traceparent"), job.get("tracestate")) + log("info", "job_rescheduled", attachment_id=str(att_id), next_attempt=nxt, delay_seconds=round(delay, 2), reason=reason) + + async def publish_result(self, row, job, variants): + """Одна транзакция: варианты (идемпотентно) + ready. Только владелец lease может опубликовать результат.""" + async with self.db.conn() as c: + async with c.transaction(): + owner = await c.fetchval("SELECT 1 FROM attachments WHERE id = $1 AND status = 'processing' AND lease_owner = $2 " + "FOR UPDATE", row["id"], WORKER_ID) + if owner is None: + return False # lease потерян (sweeper вернул задачу): результат не публикуем, чужой повтор доделает + for v in variants: + await c.execute( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, size_bytes, width, " + "height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) " + "ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING", + row["id"], job["pipeline_version"], v["name"], v["content_type"], len(v["data"]), v["width"], v["height"], + v["sha256"], "att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"])) + await c.execute("UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = now(), updated_at = now(), " + "lease_owner = NULL, lease_expires_at = NULL WHERE id = $1", row["id"]) + await c.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt) VALUES ($1, $2, 'ready', $3)", + row["id"], uuid.UUID(job["job_id"]), job["attempt"]) + return True + + # --- обработка ---------------------------------------------------------------- + async def process(self, job): + att_id = uuid.UUID(job["attachment_id"]) + row = await self.claim(job) + if row is None: + state = await self.db.fetchval("SELECT status FROM attachments WHERE id = $1", att_id) + log("info", "job_noop", attachment_id=job["attachment_id"], status=state or "unknown") + telemetry.record_job("attachment.process.v1", "noop", 0.0) + return # ready/failed/processing у другого владельца/неизвестное вложение: идемпотентный no-op, ack + started = time.perf_counter() + outcome = "ready" + try: + await self.fault("after_claim") + stream, _ = await self.s3.open_stream(row["staging_key"]) + spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024) + try: + try: + while True: + chunk = await stream.read(256 * 1024) + if not chunk: + break + spool.write(chunk) + finally: + stream.close() + loop = asyncio.get_running_loop() + variants = await asyncio.wait_for(loop.run_in_executor(self.pool, pipeline.process, spool), config.JOB_TIMEOUT) + finally: + spool.close() + # оригинал и варианты в конечный prefix: ключи детерминированы, повторная запись перезаписывает те же байты + await self.copy_original(row) + for v in variants: + await self.s3.put_bytes("att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]), v["data"], v["content_type"]) + await self.fault("after_s3_before_commit") + if not await self.publish_result(row, job, variants): + log("warning", "lease_lost", attachment_id=job["attachment_id"]) + return + await self.fault("after_commit_before_ack") + log("info", "job_ready", attachment_id=job["attachment_id"], attempt=job["attempt"], + duration_ms=round((time.perf_counter() - started) * 1000, 1)) + try: + await self.s3.delete(row["staging_key"]) # staging безопасно удалять после COMMIT результата + except Exception as exc: + log("warning", "staging_cleanup_failed", error_type=type(exc).__name__) + except pipeline.PermanentError as exc: + outcome = "failed" + await self.fail(att_id, job, exc.code) + log("warning", "job_failed", attachment_id=job["attachment_id"], error_code=exc.code) + except MemoryError: + outcome = "failed" + await self.fail(att_id, job, "processing_failed") + log("error", "job_failed_oom", attachment_id=job["attachment_id"]) + except asyncio.CancelledError: + raise + except InjectedCrash: + raise + except Exception as exc: # S3/БД/таймаут/неожиданное: временная ошибка + outcome = "retried" + await self.reschedule(att_id, job, type(exc).__name__) + finally: + ttr = None + if outcome == "ready": + ttr = await self.db.fetchval("SELECT extract(epoch FROM ready_at - accepted_at) FROM attachments WHERE id = $1 " + "AND status = 'ready'", att_id) + telemetry.record_job("attachment.process.v1", outcome, time.perf_counter() - started, float(ttr) if ttr is not None else None) + + async def copy_original(self, row): + stream, size = await self.s3.open_stream(row["staging_key"]) + spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024) + try: + while True: + chunk = await stream.read(256 * 1024) + if not chunk: + break + spool.write(chunk) + stream.close() + spool.seek(0) + await self.s3.put(row["s3_key"], spool, row["content_type"], size) + finally: + spool.close() + + async def fault(self, point): + """Точки внедрения сбоев для экспериментов (FAULT_INJECTION=crash:<точка> => процесс умирает мгновенно).""" + if "crash_once:" + point in config.FAULT_INJECTION: + marker = "/tmp/fault-%s.done" % point # сохраняется при рестарте того же контейнера -> сбой ровно один раз + if not os.path.exists(marker): + open(marker, "w").close() + log("error", "fault_crash_once", point=point) + os._exit(137) + if "crash:" + point in config.FAULT_INJECTION: + log("error", "fault_crash", point=point) + os._exit(137) + if "fail:" + point in config.FAULT_INJECTION: + raise OSError("injected failure at " + point) + + async def on_message(self, message): + try: + job = json.loads(message.body) + if (job.get("schema_version") != 1 or job.get("type") != "attachment.process.v1" + or not isinstance(job.get("attempt"), int) or not isinstance(job.get("pipeline_version"), int)): + raise ValueError("bad job") + uuid.UUID(job["attachment_id"]), uuid.UUID(job["job_id"]) + except Exception: + log("error", "job_unreadable") + await message.reject(requeue=False) # -> DLQ + return + self.busy += 1 + carrier = {k: job[k] for k in ("traceparent", "tracestate") if k in job} + try: + with telemetry.span("attachment.process process", "consumer", + {"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process", + "messaging.operation": "process", "job.type": job["type"], "job.attempt": job["attempt"]}, carrier): + await self.process(job) + await message.ack() + except InjectedCrash: + raise + except Exception as exc: + log("error", "job_unhandled", error_type=type(exc).__name__) + await message.nack(requeue=True) + finally: + self.busy -= 1 + + # --- sweeper ------------------------------------------------------------------- + async def sweep_once(self): + """Истёкший lease -> queued (новая попытка) или failed/retry_exhausted; чистка старого staging.""" + rows = await self.db.fetch( + "WITH exp AS (SELECT id FROM attachments WHERE status = 'processing' AND lease_expires_at < now() FOR UPDATE SKIP LOCKED), " + "u AS (UPDATE attachments a SET status = CASE WHEN a.attempt >= $1 THEN 'failed' ELSE 'queued' END, " + "error_code = CASE WHEN a.attempt >= $1 THEN 'retry_exhausted' ELSE NULL END, lease_owner = NULL, lease_expires_at = NULL, " + "updated_at = now() FROM exp WHERE a.id = exp.id RETURNING a.id, a.status, a.pipeline_version, a.attempt, a.job_id), " + "o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) " + "SELECT COALESCE(job_id, gen_random_uuid()), id, pipeline_version, attempt + 1, " + "(SELECT traceparent FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1), " + "(SELECT tracestate FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1) FROM u WHERE status = 'queued' RETURNING 1), " + "e AS (INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, job_id, 'lease_expired', attempt, status FROM u RETURNING 1) " + "SELECT id, status FROM u", config.MAX_ATTEMPTS) + for r in rows: + log("warning", "lease_expired", attachment_id=str(r["id"]), new_status=r["status"]) + if rows: + self.relay.kick() + + async def sweeper(self): + while not self.stopping: + try: + await self.sweep_once() + except Exception as exc: + log("warning", "sweeper_failed", error_type=type(exc).__name__) + await asyncio.sleep(config.SWEEP_INTERVAL) + + async def staging_janitor(self): + """Удаляет staging для ready (если не удалён после COMMIT) и для failed старше retention; текущую работу не трогает.""" + while not self.stopping: + await asyncio.sleep(max(30, config.SWEEP_INTERVAL * 6)) + try: + rows = await self.db.fetch( + "SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (status = 'ready' OR " + "(status = 'failed' AND updated_at < now() - make_interval(secs => $1))) LIMIT 200", + float(config.STAGING_FAILED_RETENTION_SECONDS)) + for r in rows: + await self.s3.delete(r["staging_key"]) + await self.db.execute("UPDATE attachments SET staging_key = NULL WHERE id = $1", r["id"]) + except Exception as exc: + log("warning", "staging_janitor_failed", error_type=type(exc).__name__) + + # --- жизненный цикл ------------------------------------------------------------ + async def connect_broker(self): + while not self.stopping: + try: + await self.broker.ensure(prefetch=config.WORKER_CONCURRENCY) + return + except Exception as exc: + log("warning", "broker_unavailable", error_type=type(exc).__name__) + await asyncio.sleep(2) + + async def consume(self): + while not self.stopping: + await self.connect_broker() + try: + queue = await self.broker.channel.declare_queue(QUEUE, durable=True, passive=True) + tag = await queue.consume(self.on_message) + log("info", "consuming", worker=WORKER_ID, prefetch=config.WORKER_CONCURRENCY) + while not self.stopping and not self.broker.channel.is_closed: + await asyncio.sleep(1) + if self.stopping: + await queue.cancel(tag) + return + except Exception as exc: + log("warning", "consume_failed", error_type=type(exc).__name__) + await asyncio.sleep(2) + + async def run(self): + telemetry.setup(config.SERVICE_NAME) + telemetry.process_metrics() + await self.db.start() + await self.s3.start() + log("info", "worker_started", worker=WORKER_ID, concurrency=config.WORKER_CONCURRENCY, lease=config.LEASE_SECONDS, + max_attempts=config.MAX_ATTEMPTS) + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, self.request_stop) + except NotImplementedError: + signal.signal(sig, lambda *_: self.request_stop()) + self.tasks = [asyncio.create_task(self.consume()), asyncio.create_task(self.relay.run()), + asyncio.create_task(self.sweeper()), asyncio.create_task(self.staging_janitor())] + beat = 0.0 + while not self.stopping: + if time.monotonic() - beat > 5: # heartbeat для HEALTHCHECK контейнера + beat = time.monotonic() + try: + with open("/tmp/worker.alive", "w") as fh: + fh.write(str(time.time())) + except OSError: + pass + await asyncio.sleep(0.5) + # graceful: перестаём брать новые задания, ждём текущие до GRACEFUL_SHUTDOWN_SECONDS + deadline = time.monotonic() + config.GRACEFUL_SHUTDOWN_SECONDS + while self.busy and time.monotonic() < deadline: + await asyncio.sleep(0.2) + self.relay.stop() + for t in self.tasks: + t.cancel() + await self.broker.stop() + await self.s3.stop() + await self.db.stop() + log("info", "worker_stopped", unfinished=self.busy) + telemetry.shutdown() + + def request_stop(self): + log("info", "worker_stopping") + self.stopping = True + + +class InjectedCrash(Exception): + pass + + +def main(): + asyncio.run(Worker().run()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 0000000..9e1d9e3 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,32 @@ +# Собственные проверки ЛР6 + +Запустите стенд по REPORT.md. Python 3.11+ с `python -m pip install -r research/requirements.txt` включает Pillow для пиксельных проверок. Публичным tests/ достаточно Python 3.9+. В Windows используйте `PYTHONUTF8=1`. + +Все сценарии выполняются **последовательно** на учебном стенде: часть тестов явно останавливает сервисы, внедряет отказы и восстанавливает настройки. Volumes не удаляются. Профиль для проверок наблюдаемости — OTEL_ENABLED=1, head sampling 1.0, обычный collector.yaml. + +```sh +python scripts/check_contract.py +python scripts/validate_openapi.py # предварительно установить requirements-dev.txt +python tests/smoke.py --base-url http://127.0.0.1:8080 +for suite in test_api test_idempotency test_attachments test_topology test_pipeline test_observability test_observability_retry test_reconcile5 test_resilience5 test_expiry test_dependencies; do + python -m unittest discover -s student-tests -p "$suite.py" -v || exit 1 +done +python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080 +python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 +sh scripts/restore-replicas.sh +``` + +В Windows публичным resilience передаются restart.cmd / stop-one.cmd; восстановление API — `docker compose up -d --wait --no-recreate --no-deps api`. Failover повторяется с восстановлением, чтобы проверить обе реплики. Фактические остановленные контейнеры и перезапуск всех сервисов сохранены в evidence/recheck_2026-11-12/. + +Набор содержит 66 тестов: API 12, идемпотентность 9, вложения 11, топология 6, pipeline 9, наблюдаемость 5, retry trace 1, reconciler 2, отказы pipeline 7, TTL 2, зависимости 2. Проверки наблюдаемости ищут реальные metrics/logs/traces, связь API → producer → consumer → S3/DB, retry в той же трассе, низкую кардинальность и отсутствие синтетических секретов в сигналах. + +Исследования для выбранного трека (запускать после тестов, не одновременно): + +```sh +mkdir -p evidence/my-run +python research/incidents.py --out evidence/my-run +python research/obs_cost.py --out evidence/my-run --profiles off on100 head10 tail --repeat 2 +python research/plot_obs_cost.py --out evidence/my-run +``` + +Четыре контролируемых инцидента проверяют firing/resolved; off/on100/head10/tail сравниваются при одинаковых ресурсах, 64 клиентах, 8 процессах, 5 с прогрева и 30 с измерения (2 повтора). Эти прогоны занимают около 20 минут. Секреты создаются scripts/init-secrets.sh и в Git не входят. Все команды и реальные результаты текущей проверки — evidence/checks_2026-11-12.txt и evidence/recheck_2026-11-12/. diff --git a/student-tests/fixtures/alpha.png b/student-tests/fixtures/alpha.png new file mode 100644 index 0000000..975b70d Binary files /dev/null and b/student-tests/fixtures/alpha.png differ diff --git a/student-tests/fixtures/exif_rot6.jpg b/student-tests/fixtures/exif_rot6.jpg new file mode 100644 index 0000000..4443f10 Binary files /dev/null and b/student-tests/fixtures/exif_rot6.jpg differ diff --git a/student-tests/fixtures/poison.png b/student-tests/fixtures/poison.png new file mode 100644 index 0000000..231a03d Binary files /dev/null and b/student-tests/fixtures/poison.png differ diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000..6b1f3ef Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/fixtures/sample.png b/student-tests/fixtures/sample.png new file mode 100644 index 0000000..e7c9207 Binary files /dev/null and b/student-tests/fixtures/sample.png differ diff --git a/student-tests/fixtures/small.png b/student-tests/fixtures/small.png new file mode 100644 index 0000000..caadb26 Binary files /dev/null and b/student-tests/fixtures/small.png differ diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..46e5e76 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,208 @@ +"""Собственные тесты lab2 против запущенного стенда (docker compose up -d). + + python -m unittest discover -s student-tests -v # BASE_URL по умолчанию http://127.0.0.1:8080 +Тесты используют только синтетические данные и ничего не удаляют. Проверки expiry и недоступности +зависимостей лежат отдельно (test_expiry.py, test_dependencies.py): они управляют контейнерами. +""" +import base64 +import json +import os +import subprocess +import unittest +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import quote + +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +ROOT = os.path.join(os.path.dirname(__file__), "..") + + +def call(method, path, body=None, token=None, headers=None, raw=None): + h = dict(headers or {}) + if method == "POST" and path.endswith("/messages") and headers is None: + h["Idempotency-Key"] = uuid.uuid4().hex # lab3: ключ обязателен; тесты без ключа — в test_idempotency.py + data = raw + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h.setdefault("Content-Type", "application/json") + if token: + h["Authorization"] = "Bearer " + token + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + status, hdrs, payload = resp.status, resp.headers, resp.read() + except urllib.error.HTTPError as err: + status, hdrs, payload = err.code, err.headers, err.read() + return status, hdrs, (json.loads(payload) if payload else None) + + +def basic(username, password): + return {"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode("utf-8")).decode()} + + +def register(prefix="u"): + name = "%s_%s" % (prefix, uuid.uuid4().hex[:12]) + password = "Pa55:word-" + uuid.uuid4().hex # двоеточие в пароле допустимо + status, _, user = call("POST", "/api/v1/users", {"username": name, "display_name": "Имя " + name, "password": password}) + assert status == 201, user + status, _, session = call("POST", "/api/v1/auth/sessions", headers=basic(name, password)) + assert status == 201, session + return {"user": user, "password": password, "token": session["token"], "session": session} + + +def psql(sql): + out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", + "-d", "messenger", "-Atc", sql], cwd=ROOT, capture_output=True, text=True, check=True) + return out.stdout.strip() + + +class AuthTest(unittest.TestCase): + def assertError(self, result, status, code): + self.assertEqual(result[0], status, result[2]) + self.assertEqual(result[2]["error"]["code"], code) + self.assertEqual(result[2]["error"]["request_id"], result[1]["X-Request-Id"]) + + def test_register_validation(self): + base = {"username": "val_" + uuid.uuid4().hex[:8], "display_name": "x", "password": "p" * 12} + for patch in ({"password": "short"}, {"password": "p" * 129}, {"password": 12345678901234}, {"extra": 1}, + {"username": "BAD"}, {"display_name": " "}): + body = dict(base, **patch) + self.assertError(call("POST", "/api/v1/users", body), 400, "invalid_request") + missing = {k: v for k, v in base.items() if k != "password"} + self.assertError(call("POST", "/api/v1/users", missing), 400, "invalid_request") + self.assertEqual(call("POST", "/api/v1/users", dict(base, password="я" * 12))[0], 201) # 12 code points + + def test_password_is_not_exposed_or_stored_plain(self): + actor = register() + self.assertEqual(set(actor["user"]), {"id", "username", "display_name", "created_at"}) + stored = psql("SELECT password_hash FROM users WHERE username = '%s'" % actor["user"]["username"]) + self.assertTrue(stored.startswith("$argon2id$"), stored[:12]) + self.assertNotIn(actor["password"], stored) + + def test_wrong_password_and_unknown_user_are_indistinguishable(self): + actor = register() + wrong = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], "wrong-password-123")) + unknown = call("POST", "/api/v1/auth/sessions", headers=basic("nobody_" + uuid.uuid4().hex[:8], "wrong-password-123")) + for r in (wrong, unknown): + self.assertError(r, 401, "unauthorized") + self.assertEqual(r[1]["WWW-Authenticate"], 'Basic realm="messenger", charset="UTF-8"') + self.assertEqual(wrong[2]["error"]["message"], unknown[2]["error"]["message"]) + for bad in ({}, {"Authorization": "Basic !!!"}, {"Authorization": "Basic " + base64.b64encode(b"nocolon").decode()}, + {"Authorization": "Bearer abc"}): + self.assertEqual(call("POST", "/api/v1/auth/sessions", headers=bad)[0], 401) + + def test_bearer_rules(self): + actor = register() + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[2], actor["user"]) + for headers in ({}, basic(actor["user"]["username"], actor["password"]), {"X-User-Id": actor["user"]["id"]}, + {"Authorization": "Bearer " + "A" * 43}, {"Authorization": "Bearer short"}): + r = call("GET", "/api/v1/users/me", headers=headers) + self.assertError(r, 401, "unauthorized") + self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"') + other = register() + me = call("GET", "/api/v1/users/me", token=actor["token"], headers={"X-User-Id": other["user"]["id"]})[2] + self.assertEqual(me["id"], actor["user"]["id"]) # X-User-Id игнорируется + + def test_session_shape_entropy_and_logout(self): + actor = register() + s = actor["session"] + self.assertEqual(s["token_type"], "Bearer") + self.assertGreaterEqual(len(s["token"]), 43) # 32 байта в base64url = 256 бит + self.assertNotIn(actor["user"]["id"], s["token"]) + self.assertNotEqual(register()["token"], s["token"]) + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"])[0], 204) + self.assertError(call("GET", "/api/v1/users/me", token=s["token"]), 401, "unauthorized") + self.assertError(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"]), 401, "unauthorized") + + def test_logout_revokes_in_database_and_cache(self): + actor = register() + call("GET", "/api/v1/users/me", token=actor["token"]) # прогреваем кэш + call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"]) + revoked = psql("SELECT count(*) FROM sessions s JOIN users u ON u.id = s.user_id " + "WHERE u.username = '%s' AND s.revoked_at IS NOT NULL" % actor["user"]["username"]) + self.assertEqual(revoked, "1") + + def test_token_only_stored_as_hash(self): + actor = register() + count = psql("SELECT count(*) FROM sessions WHERE token_hash = '\\x00'::bytea OR encode(token_hash,'escape') = '%s'" % actor["token"]) + self.assertEqual(count, "0") + + def test_concurrent_registration_of_same_username(self): + name = "race_" + uuid.uuid4().hex[:10] + body = {"username": name, "display_name": "x", "password": "racing-password-1"} + with ThreadPoolExecutor(12) as pool: + codes = sorted(pool.map(lambda _: call("POST", "/api/v1/users", body)[0], range(12))) + self.assertEqual(codes, [201] + [409] * 11) + self.assertEqual(psql("SELECT count(*) FROM users WHERE username = '%s'" % name), "1") + + +class DataTest(unittest.TestCase): + def test_chat_creation_is_atomic(self): + a = register() + before = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)") + ghost = str(uuid.uuid4()) + r = call("POST", "/api/v1/chats", {"title": "откат", "member_ids": [a["user"]["id"], ghost]}, token=a["token"]) + self.assertEqual(r[0], 400) + after = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)") + self.assertEqual(before, after, "после неудачного создания не должно остаться чата или участников") + ok = call("POST", "/api/v1/chats", {"title": "ок", "member_ids": [a["user"]["id"]]}, token=a["token"]) + self.assertEqual(ok[0], 201) + + def test_foreign_chat_empty_message_bad_json(self): + a, b, eve = register(), register(), register() + chat = call("POST", "/api/v1/chats", {"title": "ЛС", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2] + path = "/api/v1/chats/%s/messages" % chat["id"] + self.assertEqual(call("POST", path, {"text": "чужое"}, eve["token"])[0], 404) + self.assertEqual(call("GET", path, token=eve["token"])[0], 404) + self.assertEqual(call("POST", path, {"text": " "}, a["token"])[0], 400) + self.assertEqual(call("POST", path, raw=b"{oops", token=a["token"], headers={"Content-Type": "application/json"})[0], 400) + self.assertEqual(call("POST", path, {"text": "x", "sender_id": b["user"]["id"]}, a["token"])[0], 400) + sent = call("POST", path, {"text": "Привет 👋"}, a["token"])[2] + self.assertEqual(sent["sender_id"], a["user"]["id"]) + + def test_multi_page_history_and_cursor_binding(self): + a, b = register(), register() + chat = call("POST", "/api/v1/chats", {"title": "стр", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2] + other = call("POST", "/api/v1/chats", {"title": "др", "member_ids": [a["user"]["id"]]}, a["token"])[2] + path = "/api/v1/chats/%s/messages" % chat["id"] + sent = [call("POST", path, {"text": "m%d" % i}, (a if i % 2 else b)["token"])[2] for i in range(23)] + walked, cursor, pages = [], None, 0 + while True: + q = "?limit=%d" % (5 if pages % 2 == 0 else 8) + ("&cursor=" + quote(cursor, safe="") if cursor else "") + page = call("GET", path + q, token=b["token"])[2] + walked += page["items"] + pages += 1 + cursor = page["next_cursor"] + if not cursor: + break + self.assertEqual(walked, sent) + self.assertGreaterEqual(pages, 3) + first = call("GET", path + "?limit=1", token=a["token"])[2]["next_cursor"] + r = call("GET", "/api/v1/chats/%s/messages?cursor=%s" % (other["id"], quote(first, safe="")), token=a["token"]) + self.assertEqual(r[0], 400) + + def test_parallel_writes_from_many_clients(self): + users = [register() for _ in range(5)] + chat = call("POST", "/api/v1/chats", {"title": "гонки", "member_ids": [u["user"]["id"] for u in users]}, users[0]["token"])[2] + path = "/api/v1/chats/%s/messages" % chat["id"] + + def client(u): + return [call("POST", path, {"text": "%s-%d" % (u["user"]["username"], i)}, u["token"])[0] for i in range(20)] + + with ThreadPoolExecutor(5) as pool: + codes = [c for part in pool.map(client, users) for c in part] + self.assertEqual(codes, [201] * 100) + total, cursor = [], None + while True: + page = call("GET", path + "?limit=100" + ("&cursor=" + quote(cursor, safe="") if cursor else ""), token=users[0]["token"])[2] + total += page["items"] + cursor = page["next_cursor"] + if not cursor: + break + self.assertEqual(len({m["id"] for m in total}), 100) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..b7fcf4b --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,205 @@ +"""Собственные тесты вложений lab6 (негативные сценарии и права). Запуск: + python -m unittest discover -s student-tests -p "test_attachments.py" -v (BASE_URL по умолчанию http://127.0.0.1:8080) +""" +import hashlib +import json +import os +import socket +import struct +import sys +import time +import unittest +import urllib.error +import urllib.parse +import urllib.request +import uuid +import zlib + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import BASE, ROOT, call, psql, register # noqa: E402 + +FIX = os.path.join(os.path.dirname(__file__), "fixtures") +PNG = open(os.path.join(FIX, "sample.png"), "rb").read() +JPG = open(os.path.join(FIX, "sample.jpg"), "rb").read() + + +def upload(token, chat_id, data, ctype="image/png", filename="sample.png", raw_filename=None): + qs = raw_filename if raw_filename is not None else "filename=" + filename + req = urllib.request.Request("%s/api/v1/chats/%s/attachments?%s" % (BASE, chat_id, qs), data=data, method="POST", + headers={"Authorization": "Bearer " + token, "Content-Type": ctype}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.headers, json.loads(r.read()) + except urllib.error.HTTPError as err: + body = err.read() + return err.code, err.headers, (json.loads(body) if body else None) + + +def download(token, att_id, suffix="/content"): + req = urllib.request.Request("%s/api/v1/attachments/%s%s" % (BASE, att_id, suffix), + headers={"Authorization": "Bearer " + token if token else ""}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.headers, r.read() + except urllib.error.HTTPError as err: + return err.code, err.headers, err.read() + + +def wait_ready(token, att_id, seconds=60): + """lab6: загрузка возвращает 202 queued; готовность определяется опросом метаданных.""" + deadline = time.time() + seconds + while time.time() < deadline: + code, _, body = download(token, att_id, "") + att = json.loads(body) + if att["status"] == "ready": + return att + assert att["status"] != "failed", att + time.sleep(0.3) + raise AssertionError("not ready") + + +def png_chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def bomb_png(width=20000, height=20000): + """Корректный по структуре PNG, заявляющий 400 млн пикселей, без данных изображения (отклоняется по заголовку).""" + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", ihdr) + png_chunk(b"IDAT", zlib.compress(b"\x00" * 16)) + png_chunk(b"IEND", b"") + + +class AttachmentTest(unittest.TestCase): + def setUp(self): + self.a, self.b, self.eve = register("att"), register("att"), register("att") + self.chat = call("POST", "/api/v1/chats", {"title": "файлы", "member_ids": [self.a["user"]["id"], self.b["user"]["id"]]}, self.a["token"])[2] + self.other = call("POST", "/api/v1/chats", {"title": "другой", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2] + + def assertError(self, result, status, code=None): + self.assertEqual(result[0], status, result[2]) + if code: + self.assertEqual(result[2]["error"]["code"], code) + + def test_png_and_jpeg_roundtrip_with_checksum(self): + for data, ctype, name in ((PNG, "image/png", "фото.png"), (JPG, "image/jpeg", "photo.jpg")): + status, headers, att = upload(self.a["token"], self.chat["id"], data, ctype, urllib.parse.quote(name)) + self.assertEqual(status, 202, att) + self.assertEqual((att["status"], att["variants"], att["error_code"]), ("queued", [], None)) + wait_ready(self.b["token"], att["id"]) + self.assertEqual((att["size_bytes"], att["sha256"], att["content_type"]), (len(data), hashlib.sha256(data).hexdigest(), ctype)) + self.assertEqual((att["chat_id"], att["owner_id"], att["filename"]), (self.chat["id"], self.a["user"]["id"], name)) + code, hdr, body = download(self.b["token"], att["id"]) # другой участник чата + self.assertEqual((code, body, hdr.get_content_type()), (200, data, ctype)) + self.assertEqual(download(self.b["token"], att["id"], "")[0], 200) + + def test_too_large_is_413_and_nothing_is_stored(self): + before = psql("SELECT count(*) FROM attachments") + r = upload(self.a["token"], self.chat["id"], PNG[:8] + b"\x00" * (10 * 1024 * 1024 + 1)) + self.assertError(r, 413, "payload_too_large") + self.assertEqual(psql("SELECT count(*) FROM attachments"), before) + + def test_corrupted_and_spoofed_files_are_415_not_500(self): + cases = { + "truncated png": (PNG[: len(PNG) // 2], "image/png"), + "garbage png": (b"\x89PNG\r\n\x1a\n" + b"\x00" * 200, "image/png"), + "random bytes": (os.urandom(500), "image/png"), + "truncated jpeg": (JPG[: len(JPG) // 2], "image/jpeg"), + "garbage jpeg": (b"\xff\xd8\xff\xe0" + b"\x00" * 200, "image/jpeg"), + "png as jpeg": (PNG, "image/jpeg"), + "jpeg as png": (JPG, "image/png"), + "text as png": (b"hello, i am not an image", "image/png"), + "gif": (b"GIF89a" + b"\x00" * 50, "image/png"), + } + for name, (data, ctype) in cases.items(): + with self.subTest(name): + self.assertError(upload(self.a["token"], self.chat["id"], data, ctype), 415, "unsupported_media_type") + self.assertError(upload(self.a["token"], self.chat["id"], PNG, "text/plain"), 415) + self.assertError(upload(self.a["token"], self.chat["id"], PNG, "multipart/form-data; boundary=x"), 415) + + def test_decompression_bomb_header_is_413(self): + self.assertError(upload(self.a["token"], self.chat["id"], bomb_png()), 413, "payload_too_large") + # 6000x6600 = 39.6 млн пикселей < лимита: по размерам допустимо, но данных изображения нет -> файл повреждён (415) + self.assertEqual(upload(self.a["token"], self.chat["id"], bomb_png(6000, 6600))[0], 415) + + def test_dangerous_filenames_are_400(self): + bad = ["a%2Fb.png", "..%5Cx.png", "x%00.png", "x%0A.png", "x%7F.png", "", "%20%20", "n" * 256] + for name in bad: + with self.subTest(name): + self.assertError(upload(self.a["token"], self.chat["id"], PNG, raw_filename="filename=" + name), 400, "invalid_request") + self.assertError(upload(self.a["token"], self.chat["id"], PNG, raw_filename=""), 400) + self.assertEqual(upload(self.a["token"], self.chat["id"], PNG, raw_filename="filename=" + "n" * 255)[0], 202) + + def test_empty_body_is_400(self): + self.assertError(upload(self.a["token"], self.chat["id"], b""), 400) + + def test_foreign_user_cannot_upload_read_or_download(self): + self.assertError(upload(self.eve["token"], self.chat["id"], PNG), 404) + att = upload(self.a["token"], self.chat["id"], PNG)[2] + wait_ready(self.a["token"], att["id"]) + for suffix in ("", "/content"): + self.assertEqual(download(self.eve["token"], att["id"], suffix)[0], 404) + self.assertEqual(download(None, att["id"], suffix)[0], 401) + self.assertEqual(download(self.a["token"], str(uuid.uuid4()))[0], 404) + self.assertEqual(download(self.a["token"], "not-a-uuid")[0], 400) + self.assertEqual(download(self.a["token"], att["id"], "/content?variant=thumbnail")[0], 200) # после ready вариант доступен + self.assertEqual(download(self.a["token"], att["id"], "/content?variant=bogus")[0], 400) + + def test_message_with_attachments_and_chat_binding(self): + att = upload(self.a["token"], self.chat["id"], PNG)[2] + foreign_chat_att = upload(self.a["token"], self.other["id"], JPG, "image/jpeg")[2] + path = "/api/v1/chats/%s/messages" % self.chat["id"] + sent = call("POST", path, {"text": "", "attachment_ids": [att["id"]]}, self.a["token"]) + self.assertEqual(sent[0], 201, sent[2]) + self.assertEqual(sent[2]["attachment_ids"], [att["id"]]) + self.assertEqual(call("POST", path, {"text": "ещё раз то же вложение", "attachment_ids": [att["id"]]}, self.b["token"])[0], 201) + listed = call("GET", path + "?limit=10", token=self.b["token"])[2]["items"] + self.assertEqual([m["attachment_ids"] for m in listed], [[att["id"]], [att["id"]]]) + # вложение другого чата / несуществующее -> 404; дубликаты, >10, пустое сообщение без вложений -> 400 + self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [foreign_chat_att["id"]]}, self.a["token"])[0], 404) + self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, self.a["token"])[0], 404) + self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [att["id"], att["id"]]}, self.a["token"])[0], 400) + self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, self.a["token"])[0], 400) + self.assertEqual(call("POST", path, {"text": " ", "attachment_ids": []}, self.a["token"])[0], 400) + self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": "nope"}, self.a["token"])[0], 400) + # чужой пользователь не может сослаться на вложение + self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [att["id"]]}, self.eve["token"])[0], 404) + + def test_idempotent_replay_with_attachments(self): + att = upload(self.a["token"], self.chat["id"], PNG)[2] + path = "/api/v1/chats/%s/messages" % self.chat["id"] + key = uuid.uuid4().hex + h = {"Idempotency-Key": key} + first = call("POST", path, {"text": "t", "attachment_ids": [att["id"]]}, self.a["token"], headers=h) + again = call("POST", path, {"attachment_ids": [att["id"]], "text": "t"}, self.a["token"], headers=h) # другой порядок ключей + self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2])) + self.assertEqual(call("POST", path, {"text": "t"}, self.a["token"], headers=h)[0], 409) # другое тело + text_key = uuid.uuid4().hex + m1 = call("POST", path, {"text": "без вложений"}, self.a["token"], headers={"Idempotency-Key": text_key}) + m2 = call("POST", path, {"text": "без вложений", "attachment_ids": []}, self.a["token"], headers={"Idempotency-Key": text_key}) + self.assertEqual(m1[2], m2[2]) # отсутствие == [] + self.assertEqual(m1[2]["attachment_ids"], []) + + def test_aborted_upload_leaves_nothing(self): + before = (psql("SELECT count(*) FROM attachments")) + host, port = BASE.split("//")[1].split(":") + with socket.create_connection((host, int(port))) as s: + s.sendall(("POST /api/v1/chats/%s/attachments?filename=a.png HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer %s\r\n" + "Content-Type: image/png\r\nContent-Length: 200000\r\n\r\n" % (self.chat["id"], self.a["token"])).encode() + PNG[:5000]) + import time + time.sleep(1) + self.assertEqual(psql("SELECT count(*) FROM attachments"), before) + + def test_bucket_is_private_outside_the_api(self): + att = upload(self.a["token"], self.chat["id"], PNG)[2] + wait_ready(self.a["token"], att["id"]) + key = psql("SELECT s3_key FROM attachments WHERE id = '%s'" % att["id"]) + import subprocess + code = ("import urllib.request,urllib.error,sys\n" + "try:\n urllib.request.urlopen('http://seaweedfs:8333/messenger-attachments/%s', timeout=5); print('OPEN')\n" + "except urllib.error.HTTPError as e: print(e.code)\n" % key) + out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code], cwd=ROOT, + capture_output=True, text=True).stdout.strip() + self.assertEqual(out, "403", "анонимный GET объекта напрямую из S3 должен быть запрещён") + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_dependencies.py b/student-tests/test_dependencies.py new file mode 100644 index 0000000..add0711 --- /dev/null +++ b/student-tests/test_dependencies.py @@ -0,0 +1,82 @@ +"""Поведение при недоступности зависимостей: Valkey (кэш) и PostgreSQL (источник истины). + + python student-tests/test_dependencies.py +Контейнеры останавливаются и запускаются обратно (volumes сохраняются). Ожидания: + * Valkey остановлен -> кэш обходится, API работает (чтение сессии из PostgreSQL), logout работает; + * PostgreSQL остановлен -> 503 в контрактном формате с Retry-After (не 500, не «анонимный» доступ), + /health/ready = 503, запрос без токена по-прежнему 401; после запуска БД API восстанавливается сам. +""" +import os +import subprocess +import sys +import time +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import ROOT, basic, call, register # noqa: E402 + + +def compose(*args): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, capture_output=True, text=True) + + +def wait(predicate, seconds=60): + deadline = time.time() + seconds + while time.time() < deadline: + try: + if predicate(): + return + except Exception: + pass + time.sleep(0.5) + raise RuntimeError("условие не выполнено за %d с" % seconds) + + +class DependencyTest(unittest.TestCase): + def tearDown(self): + compose("start", "postgres", "valkey") + wait(lambda: call("GET", "/health/ready")[0] == 200) + + def test_valkey_down_is_bypassed(self): + actor = register("dep") + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) + compose("stop", "valkey") + started = time.time() + for _ in range(5): + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) + self.assertLess(time.time() - started, 3, "circuit breaker не должен давать каждому запросу ждать таймаут") + # вход и выход работают без кэша + s = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"])) + self.assertEqual(s[0], 201) + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s[2]["token"])[0], 204) + self.assertEqual(call("GET", "/api/v1/users/me", token=s[2]["token"])[0], 401) # отзыв виден без кэша + self.assertEqual(call("GET", "/health/ready")[0], 200) # кэш не критичен для readiness + + def test_postgres_down_gives_bounded_503_never_anonymous_access(self): + actor = register("dep") + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # токен в кэше + compose("stop", "postgres") + # Wait past HAProxy's fall=2 health checks: cover the all-unready state, + # not just the short interval while the load balancer still routes normally. + time.sleep(5) + for method, path, body, token in (("GET", "/api/v1/users/me", None, actor["token"]), + ("GET", "/api/v1/chats", None, actor["token"]), + ("POST", "/api/v1/chats", {"title": "x", "member_ids": [actor["user"]["id"]]}, actor["token"]), + ("GET", "/api/v1/users/me", None, "Z" * 43)): # неизвестный токен: кэш пуст -> БД -> 503 + r = call(method, path, body, token) + self.assertEqual(r[0], 503, (path, r)) + self.assertEqual(r[2]["error"]["code"], "unavailable") + self.assertGreaterEqual(int(r[1]["Retry-After"]), 1) + self.assertEqual(r[2]["error"]["request_id"], r[1]["X-Request-Id"]) + r = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"])) + self.assertEqual(r[0], 503) + self.assertEqual(call("GET", "/health/ready")[0], 503) + self.assertEqual(call("GET", "/health/live")[0], 200) + self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) # без токена — по-прежнему 401 + compose("start", "postgres") + wait(lambda: call("GET", "/health/ready")[0] == 200) + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # сессия пережила простой БД + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 0000000..293fd05 --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,69 @@ +"""Проверка срока жизни сессии при коротком SESSION_TTL_SECONDS (по умолчанию 2). + + python student-tests/test_expiry.py +Скрипт пересоздаёт только контейнер api с SESSION_TTL_SECONDS=2, проверяет expiry и возвращает настройки +по умолчанию. Данные и volumes не затрагиваются. +""" +import os +import subprocess +import sys +import time +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import ROOT, call, register # noqa: E402 + +TTL = 2 + + +def compose(*args, env=None): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, + capture_output=True, text=True, env=dict(os.environ, **(env or {}))) + + +def wait_ready(seconds=60): + deadline = time.time() + seconds + while time.time() < deadline: + try: + if call("GET", "/health/ready")[0] == 200: + return + except Exception: + pass + time.sleep(0.5) + raise RuntimeError("api не стал ready") + + +class ExpiryTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + compose("up", "-d", "--no-deps", "--force-recreate", "api", env={"SESSION_TTL_SECONDS": str(TTL)}) + wait_ready() + + @classmethod + def tearDownClass(cls): + compose("up", "-d", "--no-deps", "--force-recreate", "api") + wait_ready() + + def test_session_expires_after_ttl(self): + actor = register("exp") + session = actor["session"] + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # валидна, кэш прогрет + time.sleep(TTL + 1.5) + r = call("GET", "/api/v1/users/me", token=actor["token"]) + self.assertEqual(r[0], 401, "после TTL токен должен быть отвергнут (и кэшем, и БД)") + self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"') + self.assertEqual(r[2]["error"]["code"], "unauthorized") + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])[0], 401) + self.assertIn("expires_at", session) + + def test_expiry_is_not_extended_by_use(self): + actor = register("exp") + for _ in range(3): + call("GET", "/api/v1/users/me", token=actor["token"]) + time.sleep(0.5) + time.sleep(1.5) + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..e421a8e --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,121 @@ +"""Идемпотентность отправки сообщений (lab3) и распределение по репликам. Запуск: + python student-tests/test_idempotency.py (BASE_URL по умолчанию http://127.0.0.1:8080, через LB) +""" +import os +import sys +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import quote + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import call, psql, register # noqa: E402 + + +def chat_with(owner, *others): + ids = [owner["user"]["id"]] + [o["user"]["id"] for o in others] + return call("POST", "/api/v1/chats", {"title": "идемпотентность", "member_ids": ids}, owner["token"])[2] + + +def history(token, chat_id): + items, cursor = [], None + while True: + page = call("GET", "/api/v1/chats/%s/messages?limit=100%s" % (chat_id, "&cursor=" + quote(cursor, safe="") if cursor else ""), token=token)[2] + items += page["items"] + cursor = page["next_cursor"] + if not cursor: + return items + + +class IdempotencyTest(unittest.TestCase): + def setUp(self): + self.a, self.b, self.eve = register(), register(), register() + self.chat = chat_with(self.a, self.b) + self.path = "/api/v1/chats/%s/messages" % self.chat["id"] + + def post(self, actor, body, key, path=None): + return call("POST", path or self.path, body, actor["token"], headers={"Idempotency-Key": key}) + + def test_key_validation(self): + for headers in ({}, {"Idempotency-Key": ""}, {"Idempotency-Key": "a b"}, + {"Idempotency-Key": "x" * 129}, {"Idempotency-Key": "a/b"}): + h = dict(headers) + h.setdefault("Content-Type", "application/json") + r = call("POST", self.path, raw=b'{"text":"x"}', token=self.a["token"], headers=h) + self.assertEqual(r[0], 400, headers) + self.assertEqual(r[2]["error"]["code"], "invalid_request") + self.assertEqual(self.post(self.a, {"text": "ok"}, "A-z_0.9:-" + "k" * 100)[0], 201) + + def test_replay_returns_same_message_and_creates_no_duplicate(self): + key = uuid.uuid4().hex + first = self.post(self.a, {"text": "один раз"}, key) + second = self.post(self.a, {"text": "один раз"}, key) + self.assertEqual((first[0], second[0]), (201, 201)) + self.assertEqual(first[2], second[2]) + self.assertEqual(len([m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "один раз"]), 1) + + def test_different_body_same_key_is_409(self): + key = uuid.uuid4().hex + self.post(self.a, {"text": "первое"}, key) + r = self.post(self.a, {"text": "другое"}, key) + self.assertEqual(r[0], 409) + self.assertEqual(r[2]["error"]["code"], "conflict") + + def test_scope_is_user_and_chat(self): + key = uuid.uuid4().hex + m1 = self.post(self.a, {"text": "общий ключ"}, key)[2] + m2 = self.post(self.b, {"text": "общий ключ"}, key)[2] # другой пользователь + other = chat_with(self.a) + m3 = self.post(self.a, {"text": "общий ключ"}, key, "/api/v1/chats/%s/messages" % other["id"])[2] # другой чат + self.assertEqual(len({m1["id"], m2["id"], m3["id"]}), 3) + + def test_replay_rechecks_session_and_membership(self): + key = uuid.uuid4().hex + self.post(self.a, {"text": "защищено"}, key) + self.assertEqual(self.post(self.eve, {"text": "защищено"}, key)[0], 404) # посторонний + r = call("POST", self.path, {"text": "защищено"}, "Z" * 43, headers={"Idempotency-Key": key}) + self.assertEqual(r[0], 401) # недействительный токен + logout = register() + chat = chat_with(logout) + path = "/api/v1/chats/%s/messages" % chat["id"] + self.post(logout, {"text": "x"}, key, path) + call("DELETE", "/api/v1/auth/sessions/current", token=logout["token"]) + self.assertEqual(call("POST", path, {"text": "x"}, logout["token"], headers={"Idempotency-Key": key})[0], 401) + + def test_semantically_equal_body_is_replayed(self): + key = uuid.uuid4().hex + first = self.post(self.a, {"text": "эквивалент"}, key) + raw = call("POST", self.path, raw=b'{ "text" : "\u044d\u043a\u0432\u0438\u0432\u0430\u043b\u0435\u043d\u0442" }', + token=self.a["token"], headers={"Idempotency-Key": key, "Content-Type": "application/json"}) + self.assertEqual((raw[0], raw[2]), (201, first[2])) + + def test_concurrent_same_key_creates_exactly_one_message_across_replicas(self): + key = uuid.uuid4().hex + with ThreadPoolExecutor(24) as pool: + results = list(pool.map(lambda _: self.post(self.a, {"text": "гонка ключа"}, key), range(24))) + self.assertEqual({r[0] for r in results}, {201}) + self.assertEqual(len({r[2]["id"] for r in results}), 1) + instances = {r[1]["X-Instance-Id"] for r in results} + self.assertGreaterEqual(len(instances), 2, "повторы должны обслуживаться разными репликами: %s" % instances) + found = [m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "гонка ключа"] + self.assertEqual(len(found), 1) + + def test_expired_key_is_reusable_after_ttl(self): + key = uuid.uuid4().hex + first = self.post(self.a, {"text": "ttl"}, key)[2] + psql("UPDATE idempotency_keys SET created_at = now() - interval '25 hours' WHERE idem_key = '%s'" % key) + again = self.post(self.a, {"text": "ttl"}, key) + self.assertEqual(again[0], 201) + self.assertNotEqual(again[2]["id"], first["id"], "после 24 ч ключ считается новым") + + def test_session_is_shared_between_replicas_without_relogin(self): + seen = set() + for _ in range(30): + r = call("GET", "/api/v1/users/me", token=self.a["token"]) + self.assertEqual(r[0], 200) + seen.add(r[1]["X-Instance-Id"]) + self.assertGreaterEqual(len(seen), 2, seen) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_observability.py b/student-tests/test_observability.py new file mode 100644 index 0000000..0536f3a --- /dev/null +++ b/student-tests/test_observability.py @@ -0,0 +1,142 @@ +"""Проверка наблюдаемости lab6: метрики, логи, трассы, связь API -> брокер -> worker, отсутствие секретов в телеметрии. + python -m unittest discover -s student-tests -p test_observability.py -v (стенд запущен; нужен ~1 минута) +Запросы к Prometheus/Loki/Tempo выполняются изнутри сети стенда (их порты наружу не опубликованы). +""" +import json +import os +import subprocess +import sys +import time +import unittest +import urllib.parse +import uuid + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import BASE, ROOT, call, register # noqa: E402 +from test_attachments import PNG, upload # noqa: E402 +from test_pipeline import wait_status # noqa: E402 + + +def net_get(url, accept=None): + cmd = ["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus", "wget", "-q", "-O", "-"] + if accept: + cmd += ["--header", "Accept: " + accept] + out = subprocess.run(cmd + [url], cwd=ROOT, capture_output=True, text=True, encoding="utf-8") + return out.stdout + + +def prom(query): + data = json.loads(net_get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(query))) + return data["data"]["result"] + + +def loki(query, limit=200): + now = int(time.time() * 1e9) + url = "http://loki:3100/loki/api/v1/query_range?limit=%d&start=%d&end=%d&query=%s" % (limit, now - 600 * 10 ** 9, now, urllib.parse.quote(query)) + data = json.loads(net_get(url)) + return [(s["stream"], v) for s in data["data"]["result"] for v in s["values"]] + + +def tempo_trace(trace_id): + raw = net_get("http://tempo:3200/api/traces/" + trace_id, "application/json") + return json.loads(raw) if raw.strip() else None + + +def spans_of(trace): + out = [] + for batch in trace.get("batches", trace.get("resourceSpans", [])): + service = next((a["value"].get("stringValue") for a in batch["resource"]["attributes"] if a["key"] == "service.name"), None) + for scope in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", [])): + for sp in scope["spans"]: + attrs = {a["key"]: next(iter(a["value"].values())) for a in sp.get("attributes", [])} + out.append({"service": service, "name": sp["name"], "id": sp["spanId"], "parent": sp.get("parentSpanId", ""), "kind": sp.get("kind"), "attrs": attrs}) + return out + + +class ObservabilityTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.a = register("obs") + cls.chat = call("POST", "/api/v1/chats", {"title": "наблюдаемость", "member_ids": [cls.a["user"]["id"]]}, cls.a["token"])[2] + cls.trace_id = uuid.uuid4().hex + cls.secret_text = "СЕКРЕТНЫЙ-ТЕКСТ-" + uuid.uuid4().hex + # загрузка с входящим traceparent: так мы знаем trace_id заранее + import urllib.request + req = urllib.request.Request("%s/api/v1/chats/%s/attachments?filename=obs.png" % (BASE, cls.chat["id"]), data=PNG, method="POST", + headers={"Authorization": "Bearer " + cls.a["token"], "Content-Type": "image/png", + "traceparent": "00-%s-%s-01" % (cls.trace_id, uuid.uuid4().hex[:16])}) + with urllib.request.urlopen(req, timeout=30) as r: + cls.att = json.loads(r.read()) + wait_status(cls.a["token"], cls.att["id"], ("ready",), 60) + call("POST", "/api/v1/chats/%s/messages" % cls.chat["id"], {"text": cls.secret_text, "attachment_ids": [cls.att["id"]]}, cls.a["token"]) + deadline = time.time() + 40 # экспорт SDK (2 с) + пакетная обработка Collector + индексация + cls.trace = None + while time.time() < deadline: + cls.trace = tempo_trace(cls.trace_id) + if cls.trace and {s["service"] for s in spans_of(cls.trace)} >= {"messenger-api", "messenger-worker"}: + break + time.sleep(2) + + def test_one_trace_spans_api_outbox_broker_and_worker(self): + self.assertIsNotNone(self.trace, "трасса не найдена в Tempo") + spans = spans_of(self.trace) + by_name = {s["name"]: s for s in spans} + self.assertEqual({s["service"] for s in spans} & {"messenger-api", "messenger-worker"}, {"messenger-api", "messenger-worker"}) + server = by_name["HTTP POST /api/v1/chats/{chat_id}/attachments"] + producer, consumer = by_name["attachment.process publish"], by_name["attachment.process process"] + self.assertEqual((server["service"], producer["service"], consumer["service"]), ("messenger-api", "messenger-api", "messenger-worker")) + # цепочка parent-child: server (API) -> producer (publish из outbox) -> consumer (worker) + self.assertEqual(producer["parent"], server["id"]) + self.assertEqual(consumer["parent"], producer["id"]) + names = {s["name"] for s in spans} + self.assertIn("s3.put_object", names) # запись staging (API) и результатов (worker) + self.assertIn("s3.get_object", names) # чтение staging worker-ом + self.assertIn("db.connection", names) + self.assertEqual(server["attrs"].get("http.route"), "/api/v1/chats/{chat_id}/attachments") + self.assertEqual(int(server["attrs"].get("http.response.status_code")), 202) + + def test_logs_are_structured_and_linked_by_trace_id(self): + rows = loki('{service_name=~"messenger-.*"} | trace_id="%s"' % self.trace_id) + services = {stream["service_name"] for stream, _ in rows} + messages = {v[1] for _, v in rows} + self.assertIn("messenger-api", services) + self.assertIn("messenger-worker", services) + self.assertIn("job_ready", messages) + self.assertTrue(all(stream.get("span_id") for stream, _ in rows)) + self.assertTrue(all(stream.get("service_version") is not None and stream.get("service_instance_id") for stream, _ in rows)) + + def test_metrics_for_http_jobs_queue_pool_worker(self): + for query in ('http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/attachments"}', + 'messenger_jobs_processed_total{outcome="ready"}', + 'messenger_job_duration_seconds_bucket', 'messenger_job_time_to_ready_seconds_bucket', + 'messenger_queue_depth', 'messenger_queue_oldest_job_age_seconds', 'messenger_worker_capacity', 'messenger_worker_busy', + 'db_client_connections_usage{state="used"}', 'db_client_connections_waiting', 'process_memory_rss_bytes', + 'process_cpu_time_seconds_total'): + with self.subTest(query): + deadline = time.time() + 40 # экспорт SDK (5 с) + scrape Prometheus (5 с) + while not prom(query) and time.time() < deadline: + time.sleep(2) + self.assertTrue(prom(query), "нет данных: " + query) + + def test_no_high_cardinality_or_secret_labels(self): + forbidden = ("user_id", "chat_id", "message_id", "attachment_id", "request_id", "filename") + series = json.loads(net_get("http://127.0.0.1:9090/api/v1/series?match[]=%s" % urllib.parse.quote('{__name__=~"http_server.*|messenger_.*|db_client.*|process_.*"}')))["data"] + labels = {k for s in series for k in s} + self.assertFalse(labels & set(forbidden), labels & set(forbidden)) + text = json.dumps(series) + self.assertNotIn(self.a["token"], text) + route_values = {s.get("http_route") for s in series if "http_route" in s} + import re + self.assertFalse([r for r in route_values if re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-", r)], route_values) # шаблоны, не реальные UUID + + def test_secrets_and_message_bodies_are_not_exported(self): + self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.secret_text), []) + self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.a["token"]), []) + self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.a["password"]), []) + raw = json.dumps(self.trace) + for secret in (self.secret_text, self.a["token"], self.a["password"]): + self.assertNotIn(secret, raw) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_observability_retry.py b/student-tests/test_observability_retry.py new file mode 100644 index 0000000..ba2f2fd --- /dev/null +++ b/student-tests/test_observability_retry.py @@ -0,0 +1,58 @@ +"""Цепочка трассировки с повторами (уровень 4): все попытки одного задания находятся в ОДНОЙ трассе. + python student-tests/test_observability_retry.py +Worker пересоздаётся с внедрением сбоя (fail:after_claim, 3 попытки), затем возвращается к штатной конфигурации. +""" +import json +import os +import sys +import time +import unittest +import urllib.request +import uuid + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import BASE, call, register # noqa: E402 +from test_attachments import PNG # noqa: E402 +from test_observability import loki, spans_of, tempo_trace # noqa: E402 +from test_pipeline import wait_status # noqa: E402 +from test_resilience5 import compose, healthy, recreate_worker, wait # noqa: E402 + + +class RetryTraceTest(unittest.TestCase): + @classmethod + def tearDownClass(cls): + recreate_worker() + + def test_all_attempts_in_one_trace_and_logs(self): + recreate_worker({"WORKER_FAULT_INJECTION": "fail:after_claim", "MAX_ATTEMPTS": "3", "RETRY_BASE_DELAY": "0.3"}) + a = register("obsr") + chat = call("POST", "/api/v1/chats", {"title": "retry-trace", "member_ids": [a["user"]["id"]]}, a["token"])[2] + trace_id = uuid.uuid4().hex + req = urllib.request.Request("%s/api/v1/chats/%s/attachments?filename=r.png" % (BASE, chat["id"]), data=PNG, method="POST", + headers={"Authorization": "Bearer " + a["token"], "Content-Type": "image/png", + "traceparent": "00-%s-%s-01" % (trace_id, uuid.uuid4().hex[:16])}) + with urllib.request.urlopen(req, timeout=30) as r: + att = json.loads(r.read()) + failed = wait_status(a["token"], att["id"], ("failed",), 60) + self.assertEqual(failed["error_code"], "retry_exhausted") + spans, deadline = [], time.time() + 40 + while time.time() < deadline: + trace = tempo_trace(trace_id) + spans = spans_of(trace) if trace else [] + consumers = [s for s in spans if s["name"] == "attachment.process process"] + if len(consumers) >= 3: + break + time.sleep(2) + consumers = sorted([s for s in spans if s["name"] == "attachment.process process"], key=lambda s: int(s["attrs"]["job.attempt"])) + publishers = {s["id"]: s for s in spans if s["name"] == "attachment.process publish"} + self.assertEqual([int(c["attrs"]["job.attempt"]) for c in consumers], [1, 2, 3]) + for c in consumers: + self.assertIn(c["parent"], publishers, "consumer-спан каждой попытки — потомок publish-спана той же трассы") + logs = loki('{service_name="messenger-worker"} | trace_id="%s"' % trace_id) + messages = [v[1] for _, v in logs] + self.assertEqual(messages.count("job_rescheduled"), 2) + self.assertIn("job_retry_exhausted", messages) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_pipeline.py b/student-tests/test_pipeline.py new file mode 100644 index 0000000..955e922 --- /dev/null +++ b/student-tests/test_pipeline.py @@ -0,0 +1,182 @@ +"""Тесты асинхронного конвейера вложений lab6 (контракт вариантов, состояния, права). Запуск: + python -m unittest discover -s student-tests -p "test_pipeline.py" -v +Пиксельные проверки выполняются, если установлен Pillow (иначе пропускаются); размеры проверяются по заголовкам файлов. +""" +import hashlib +import io +import os +import struct +import sys +import time +import unittest +import uuid + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import call, register # noqa: E402 +from test_attachments import FIX, JPG, PNG, download, upload # noqa: E402 + +try: + from PIL import Image +except ImportError: # pragma: no cover + Image = None + + +def fixture(name): + return open(os.path.join(FIX, name), "rb").read() + + +def jpeg_size(data): + i = 2 + while i < len(data): + while data[i] == 0xFF: + i += 1 + marker = data[i] + i += 1 + if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7: + continue + length = int.from_bytes(data[i:i + 2], "big") + if marker in (0xC0, 0xC1, 0xC2): + return int.from_bytes(data[i + 5:i + 7], "big"), int.from_bytes(data[i + 3:i + 5], "big") + i += length + raise AssertionError("JPEG dimensions not found") + + +def webp_size(data): + assert data[:4] == b"RIFF" and data[8:12] == b"WEBP", "not WebP" + kind = data[12:16] + if kind == b"VP8X": + return 1 + int.from_bytes(data[24:27], "little"), 1 + int.from_bytes(data[27:30], "little") + if kind == b"VP8L": + bits = int.from_bytes(data[21:25], "little") + return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1 + if kind == b"VP8 ": + return struct.unpack(" 100x300 -> вписывается в 256: 85x256 (округление вниз) + self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"]), (85, 256)) + self.assertEqual(jpeg_size(self.fetch(att, "thumbnail")[1]), (85, 256)) + + def test_small_image_is_not_upscaled_but_crop_is_128(self): + att = self.ready(fixture("small.png"), "image/png", "small.png") + v = self.variants(att) + self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"]), (50, 30)) + self.assertEqual((v["crop"]["width"], v["crop"]["height"]), (128, 128)) + self.assertEqual((v["optimized"]["width"], v["optimized"]["height"]), (50, 30)) + + @unittest.skipIf(Image is None, "нужен Pillow") + def test_alpha_is_composited_on_white(self): + att = self.ready(fixture("alpha.png"), "image/png", "alpha.png") + thumb = Image.open(io.BytesIO(self.fetch(att, "thumbnail")[1])).convert("RGB") + left, right = thumb.getpixel((10, thumb.height // 2)), thumb.getpixel((thumb.width - 10, thumb.height // 2)) + self.assertTrue(all(c > 235 for c in left), "прозрачная область должна стать белой: %s" % (left,)) + self.assertTrue(right[0] > 200 and right[1] < 60, "непрозрачная красная область сохранена: %s" % (right,)) + + def test_message_can_reference_attachment_before_ready_and_failed_cannot(self): + status, _, att = upload(self.a["token"], self.chat["id"], PNG) + path = "/api/v1/chats/%s/messages" % self.chat["id"] + sent = call("POST", path, {"text": "", "attachment_ids": [att["id"]]}, self.a["token"]) + self.assertEqual((status, sent[0], sent[2]["attachment_ids"]), (202, 201, [att["id"]])) + poison = upload(self.a["token"], self.chat["id"], fixture("poison.png")) + self.assertEqual(poison[0], 202, poison[2]) + failed = wait_status(self.b["token"], poison[2]["id"], ("failed",)) + self.assertEqual((failed["status"], failed["error_code"], failed["variants"]), ("failed", "invalid_image", [])) + self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [failed["id"]]}, self.a["token"])[0], 409) + self.assertEqual(download(self.b["token"], failed["id"], "/content")[0], 409) + self.assertEqual(download(self.b["token"], failed["id"], "/content?variant=thumbnail")[0], 409) + + def test_content_is_409_while_not_ready_and_original_is_untouched(self): + status, _, att = upload(self.a["token"], self.chat["id"], PNG) + code = download(self.b["token"], att["id"], "/content")[0] + self.assertIn(code, (200, 409)) # быстрый worker мог успеть: 200 допустим, но не 404/500 + done = wait_status(self.b["token"], att["id"]) + self.assertEqual(download(self.b["token"], done["id"], "/content")[2], PNG) + + def test_other_variants_visible_only_to_chat_members(self): + att = self.ready(PNG) + for variant in ("thumbnail", "crop", "optimized", "original"): + self.assertEqual(download(self.eve["token"], att["id"], "/content?variant=" + variant)[0], 404) + self.assertEqual(download(self.b["token"], att["id"], "/content?variant=bogus")[0], 400) + + def test_ready_job_redelivery_does_not_duplicate_variants(self): + from test_api import psql + att = self.ready(PNG) + before = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]) + self.assertEqual(before, "3") + psql("INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES (gen_random_uuid(), '%s', 1, 2)" % att["id"]) + time.sleep(4) + self.assertEqual(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]), "3") + self.assertEqual(wait_status(self.b["token"], att["id"])["status"], "ready") + + def test_staging_is_cleaned_after_ready_and_kept_for_failed(self): + import subprocess + from test_api import ROOT, psql + att = self.ready(PNG) + poison = upload(self.a["token"], self.chat["id"], fixture("poison.png"))[2] + wait_status(self.b["token"], poison["id"], ("failed",)) + + def exists(att_id): + code = "\n".join(["import asyncio", "from app.s3 import S3", "async def m():", + " s = S3(); await s.start(); print((await s.head('staging/%s')) is not None); await s.stop()" % att_id, + "asyncio.run(m())"]) + return subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code], cwd=ROOT, + capture_output=True, text=True).stdout.strip() + self.assertEqual(exists(att["id"]), "False", "staging готового вложения должен быть удалён после COMMIT результата") + self.assertEqual(exists(poison["id"]), "True", "staging неудавшегося вложения хранится для разбора/replay (retention)") + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_reconcile5.py b/student-tests/test_reconcile5.py new file mode 100644 index 0000000..d2abceb --- /dev/null +++ b/student-tests/test_reconcile5.py @@ -0,0 +1,89 @@ +"""Reconciliation must preserve variants, in-flight effects and durable staging.""" +import hashlib +import json +import os +import subprocess +import sys +import unittest +import uuid + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import ROOT, call, register +from test_attachments import PNG, download, upload +from test_pipeline import wait_status + + +def compose(*args): + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, + check=True, capture_output=True, text=True, encoding='utf-8').stdout + + +def reconcile(prefix, grace=0): + return json.loads(compose('run', '--rm', '-T', 'api', 'python', '-m', 'app.reconcile', + '--apply', '--grace', str(grace), '--prefix', prefix)) + + +def s3_action(action, key): + code = ('import asyncio\nfrom app import config\nfrom app.s3 import S3\nasync def m():\n' + ' s=S3(); await s.start()\n' + + {'put': ' await s.put_bytes(%r, b"temporary", "application/octet-stream")\n', + 'head': ' print((await s.head(%r)) is not None)\n', + 'delete': ' await s.delete(%r)\n'}[action] % key + + ' await s.stop()\nasyncio.run(m())') + return compose('exec', '-T', 'api', 'python', '-c', code).strip() + + +class ReconcilePipelineTest(unittest.TestCase): + def setUp(self): + self.actor = register('reconcile') + self.token = self.actor['token'] + self.chat = call('POST', '/api/v1/chats', {'title': 'reconcile', 'member_ids': [self.actor['user']['id']]}, self.token)[2] + + def accept(self): + status, _, att = upload(self.token, self.chat['id'], PNG) + self.assertEqual(status, 202) + return att + + def test_ready_variants_and_uncommitted_side_effect_are_preserved(self): + att = wait_status(self.token, self.accept()['id']) + key = 'att/%s/v1/uncommitted' % att['id'] + s3_action('put', key) + try: + report = reconcile('att/%s/' % att['id']) + self.assertEqual(report['fixed'], {'deleted_orphans': 0, 'marked_failed': 0}) + self.assertEqual(report['orphan_objects_count'], 0) + self.assertEqual(report['owned_objects_skipped'], 1) + self.assertEqual(s3_action('head', key), 'True') + self.assertEqual(download(self.token, att['id'])[2], PNG) + for variant in att['variants']: + code, _, data = download(self.token, att['id'], '/content?variant=' + variant['name']) + self.assertEqual(code, 200) + self.assertEqual(hashlib.sha256(data).hexdigest(), variant['sha256']) + finally: + s3_action('delete', key) + + def test_staging_is_protected_and_unowned_orphan_obeys_grace(self): + compose('stop', 'worker') + att = None + orphan = 'staging/' + str(uuid.uuid4()) + try: + att = self.accept() + report = reconcile('staging/' + att['id']) + self.assertEqual(report['fixed'], {'deleted_orphans': 0, 'marked_failed': 0}) + self.assertEqual(s3_action('head', 'staging/' + att['id']), 'True') + s3_action('put', orphan) + fresh = reconcile(orphan, grace=900) + self.assertEqual(fresh['orphans_skipped_fresh'], 1) + self.assertEqual(fresh['fixed']['deleted_orphans'], 0) + self.assertEqual(reconcile(orphan)['fixed']['deleted_orphans'], 1) + self.assertEqual(s3_action('head', orphan), 'False') + self.assertEqual(reconcile(orphan)['fixed']['deleted_orphans'], 0) + finally: + s3_action('delete', orphan) + compose('start', 'worker') + self.assertEqual(wait_status(self.token, att['id'])['status'], 'ready') + self.assertEqual(download(self.token, att['id'])[2], PNG) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_resilience5.py b/student-tests/test_resilience5.py new file mode 100644 index 0000000..1e25e5b --- /dev/null +++ b/student-tests/test_resilience5.py @@ -0,0 +1,154 @@ +"""Отказы конвейера lab6 (окна отказа из contracts/attachment-states.md). Тесты управляют контейнерами (volumes сохраняются): + python student-tests/test_resilience5.py # ~5-7 минут + +Окна: (1) worker остановлен -> upload -> рестарт API -> worker поднят; (2) рестарт брокера с volume; (3) брокер недоступен +при приёме (outbox); (4) crash API между COMMIT и publish; (5) crash worker после записи в S3 до COMMIT результата; +(6) crash после COMMIT до ack; (7) исчерпание попыток и операторский replay. +""" +import json +import os +import subprocess +import sys +import time +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import ROOT, call, psql, register # noqa: E402 +from test_attachments import PNG, download, upload # noqa: E402 +from test_pipeline import wait_status # noqa: E402 + + +def compose(*args, env=None, check=True): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True, text=True, + env=dict(os.environ, **(env or {})), check=check).stdout + + +def wait(predicate, seconds=90, message="условие"): + deadline = time.time() + seconds + while time.time() < deadline: + try: + if predicate(): + return + except Exception: + pass + time.sleep(1) + raise AssertionError("%s не выполнено за %d с" % (message, seconds)) + + +def healthy(service): + ids = compose("ps", "-q", service).split() + return bool(ids) and all(subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", i], capture_output=True, text=True).stdout.strip() == "healthy" for i in ids) + + +def api_ready(): + return call("GET", "/health/ready")[0] == 200 + + +def recreate_worker(env=None): + compose("up", "-d", "--no-deps", "--force-recreate", "worker", env=env) + wait(lambda: healthy("worker"), 90, "worker healthy") + + +class ResilienceTest(unittest.TestCase): + @classmethod + def tearDownClass(cls): + recreate_worker() + compose("up", "-d", "--no-deps", "rabbitmq", "api") + + def setUp(self): + compose("start", "rabbitmq", "postgres", "seaweedfs", "valkey") + wait(lambda: healthy("rabbitmq") and healthy("postgres"), 90, "rabbitmq/postgres healthy") + wait(api_ready, 60, "api ready") + self.a = register("res") + self.chat = call("POST", "/api/v1/chats", {"title": "отказы", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2] + + def accept(self, data=PNG): + status, _, att = upload(self.a["token"], self.chat["id"], data) + self.assertEqual(status, 202, att) + return att + + def done(self, att, seconds=90): + ready = wait_status(self.a["token"], att["id"], ("ready",), seconds) + self.assertEqual(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]), "3") # без дублей + code, _, body = download(self.a["token"], att["id"]) + self.assertEqual((code, body), (200, PNG)) + return ready + + def test_1_worker_down_upload_api_restart_worker_up(self): + compose("stop", "worker") + att = self.accept() + sent = call("POST", "/api/v1/chats/%s/messages" % self.chat["id"], {"text": "", "attachment_ids": [att["id"]]}, self.a["token"]) + self.assertEqual(sent[0], 201) + time.sleep(3) + self.assertEqual(wait_status(self.a["token"], att["id"], ("queued",), 5)["status"], "queued") # не обрабатывается без worker + compose("restart", "api") + wait(api_ready, 60, "api ready") + compose("start", "worker") + self.done(att) + + def test_2_broker_restart_keeps_accepted_jobs(self): + compose("stop", "worker") + att = self.accept() + time.sleep(2) + compose("restart", "rabbitmq") + wait(lambda: healthy("rabbitmq"), 90, "rabbitmq healthy") + compose("start", "worker") + self.done(att) + + def test_3_broker_down_during_acceptance_outbox_delivers_later(self): + compose("stop", "rabbitmq") + att = self.accept() # 202 даже без брокера: намерение зафиксировано в outbox + self.assertEqual(psql("SELECT count(*) FROM outbox WHERE attachment_id = '%s' AND published_at IS NULL" % att["id"]), "1") + time.sleep(3) + self.assertEqual(wait_status(self.a["token"], att["id"], ("queued",), 5)["status"], "queued") + compose("start", "rabbitmq") + self.done(att) + self.assertEqual(psql("SELECT count(*) FROM outbox WHERE attachment_id = '%s' AND published_at IS NULL" % att["id"]), "0") + + def test_4_api_crash_between_commit_and_publish(self): + compose("stop", "rabbitmq") + att = self.accept() # COMMIT выполнен, publish невозможен + for cid in compose("ps", "-q", "api").split(): + subprocess.run(["docker", "kill", cid], capture_output=True) # внезапная смерть всех реплик API + compose("start", "rabbitmq") + wait(lambda: healthy("rabbitmq"), 90, "rabbitmq healthy") + compose("up", "-d", "--no-deps", "api") + wait(api_ready, 90, "api ready") + self.done(att) # relay (в worker/новом API) довёл задание без ручного вмешательства + + def test_5_worker_crash_after_s3_before_commit_lease_recovery(self): + recreate_worker({"WORKER_FAULT_INJECTION": "crash_once:after_s3_before_commit", "LEASE_SECONDS": "8"}) + att = self.accept() + ready = self.done(att, 120) + events = psql("SELECT string_agg(event, ',' ORDER BY id) FROM job_events WHERE attachment_id = '%s'" % att["id"]) + self.assertIn("lease_expired", events) + self.assertIn("ready", events) + self.assertEqual(ready["status"], "ready") + + def test_6_worker_crash_after_commit_before_ack_redelivery_is_noop(self): + recreate_worker({"WORKER_FAULT_INJECTION": "crash_once:after_commit_before_ack"}) + att = self.accept() + self.done(att, 120) + time.sleep(5) # повторная доставка: no-op с ack, вариантов по-прежнему 3 + self.assertEqual(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]), "3") + self.assertEqual(psql("SELECT status FROM attachments WHERE id = '%s'" % att["id"]), "ready") + + def test_7_retry_exhaustion_registry_and_operator_replay(self): + recreate_worker({"WORKER_FAULT_INJECTION": "fail:after_claim", "MAX_ATTEMPTS": "3", "RETRY_BASE_DELAY": "0.3"}) + att = self.accept() + failed = wait_status(self.a["token"], att["id"], ("failed",), 60) + self.assertEqual((failed["error_code"], failed["variants"]), ("retry_exhausted", [])) + events = psql("SELECT string_agg(event || ':' || coalesce(attempt::text,''), ',' ORDER BY id) FROM job_events WHERE attachment_id = '%s'" % att["id"]) + self.assertEqual(events.count("retry_scheduled"), 2, events) + registry = json.loads(compose("run", "--rm", "-T", "worker", "python", "-m", "app.replay", "list")) + self.assertIn(att["id"], [r["attachment_id"] for r in registry]) + recreate_worker() # причина устранена + result = json.loads(compose("run", "--rm", "-T", "worker", "python", "-m", "app.replay", "replay", att["id"])) + self.assertEqual(result["result"], "requeued") + self.done(att) + again = json.loads(compose("run", "--rm", "-T", "worker", "python", "-m", "app.replay", "replay", att["id"])) + self.assertEqual(again["result"], "skipped") # replay не исполняет дубль уже успешной работы + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_topology.py b/student-tests/test_topology.py new file mode 100644 index 0000000..7c39b4b --- /dev/null +++ b/student-tests/test_topology.py @@ -0,0 +1,59 @@ +"""Проверка топологии стенда (уровень 4): число реплик, отсутствие публикации портов API, лимиты ресурсов, healthcheck, +отсутствие container_name, единая публичная точка входа. Запуск: python student-tests/test_topology.py""" +import json +import subprocess +import unittest + +PROJECT = "mtusi-lab6" + + +def sh(*args): + return subprocess.run(args, capture_output=True, text=True, encoding="utf-8").stdout.strip() + + +def containers(service): + ids = sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=" + service).split() + return [json.loads(sh("docker", "inspect", i))[0] for i in ids] + + +class TopologyTest(unittest.TestCase): + def test_at_least_two_api_replicas_without_published_ports(self): + api = containers("api") + self.assertGreaterEqual(len(api), 2) + for c in api: + self.assertFalse([p for p in c["HostConfig"]["PortBindings"] or {}], "реплика API публикует порт в обход LB") + self.assertEqual(c["State"]["Health"]["Status"], "healthy") + + def test_only_lb_is_published_and_only_on_localhost(self): + published = {} + for service in ("api", "postgres", "valkey", "lb", "seaweedfs", "rabbitmq", "otel-collector", "prometheus", "loki", "tempo", "grafana"): + for c in containers(service): + for port, binds in (c["HostConfig"]["PortBindings"] or {}).items(): + published.setdefault(service, []).extend((port, b["HostIp"], b["HostPort"]) for b in binds) + self.assertEqual(sorted(published), ["grafana", "lb"], published) # lab6: LB и Grafana (UI), OTLP/метрики/админки наружу не открыты + self.assertEqual(published["lb"], [("8080/tcp", "127.0.0.1", "8080")]) + self.assertEqual(published["grafana"], [("3000/tcp", "127.0.0.1", "3000")]) + + def test_resource_limits_are_set(self): + for service in ("api", "postgres", "valkey", "lb", "seaweedfs", "rabbitmq", "otel-collector", "prometheus", "loki", "tempo", "grafana"): + for c in containers(service): + self.assertGreater(c["HostConfig"]["Memory"], 0, service) + self.assertGreater(c["HostConfig"]["NanoCpus"], 0, service) + + def test_no_fixed_container_names(self): + for c in containers("api"): + labels = c["Config"]["Labels"] + self.assertEqual(c["Name"].lstrip("/"), "%s-api-%s" % (PROJECT, labels["com.docker.compose.container-number"])) + + def test_api_runs_as_non_root(self): + for c in containers("api"): + self.assertEqual(c["Config"]["User"], "10001:10001") + + def test_stop_grace_period_covers_drain(self): + for c in containers("api"): + self.assertGreaterEqual(c["Config"].get("StopTimeout", 10), 15) + + +if __name__ == "__main__": + unittest.main()