diff --git a/.env.example b/.env.example index 2b32655..fbd7637 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,5 @@ OTEL_SERVICE_NAME=messenger-api OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +GRAFANA_PORT=3000 +PROMETHEUS_PORT=9090 diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 0000000..87430cc --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,35 @@ +name: Lab 6 application and telemetry +on: [push, pull_request] +permissions: + contents: read +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-dev.txt + - run: make check validate PYTHON=python + - name: Start disposable stack + run: | + cp .env.example .env + python scripts/setup-local-secrets.py + python scripts/init_local_broker.py + docker compose -f compose.yaml -f compose.telemetry.yaml --env-file .env up -d --build --wait --scale api=2 + - name: Verify API and telemetry rules + run: | + make test + python - <<'PY' + import json, urllib.request + url = 'http://127.0.0.1:9090/api/v1/rules' + data = json.load(urllib.request.urlopen(url, timeout=10)) + assert data['status'] == 'success' + names = {rule['name'] for group in data['data']['groups'] for rule in group['rules']} + assert {'Lab6HighHttpErrorRate', 'Lab6QueueLag'} <= names, names + PY + - name: Stop stack + if: always() + run: docker compose -f compose.yaml -f compose.telemetry.yaml --env-file .env down || true diff --git a/.gitignore b/.gitignore index b5a7189..860fedd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ # Local/private state +/docs/plans/ +/LAB_NOTES.md .env .env.* !.env.example +compose.local.yaml secrets/ certs/private/ *.key diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1ecad6d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --system --uid 10001 --home /app app +COPY --chown=app:app src/ src/ +COPY --chown=app:app migrations/ migrations/ +COPY --chown=app:app scripts/migrate.py scripts/migrate.py +USER app +EXPOSE 8080 +CMD ["python", "-m", "src.server"] diff --git a/REPORT.md b/REPORT.md index b97ae42..46c4fda 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,109 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: Самохвалов Андрей Ремович, БПИ-2302. +- Целевая оценка: 5; трек 2 «SLO и обнаружение инцидентов». Уровни 3 и 4 проверяются отдельно. +- Ветка: `solution/bpi2302-samohvalov`. Локальные `docs/plans/` и `LAB_NOTES.md` не публикуются. +- Источники: `README.md`, `COURSE.md`, `contracts/observability.md`, публичные проверки и собственная lab5. Помощь ИИ: Codex; результаты ниже получены на синтетических данных. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Compose и Python 3.9+. На хост открыты только loopback-порты API 8080, Grafana 3000 и Prometheus 9090. Секреты создаются локально с правами 0600 и не попадают в Git. -## Архитектура и решения +```sh +cp .env.example .env +python3 scripts/setup-local-secrets.py +python3 scripts/init_local_broker.py +docker compose -f compose.yaml -f compose.telemetry.yaml --env-file .env config --quiet +docker compose -f compose.yaml -f compose.telemetry.yaml --env-file .env up -d --build --wait --scale api=2 +make check +make test BASE_URL=http://127.0.0.1:8080 +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`migrate`, `s3-volume-init`, `s3-bootstrap`, `broker-init` — одноразовые роли, их успешный `Exited (0)` штатен. Остановка с сохранением томов: `docker compose -f compose.yaml -f compose.telemetry.yaml --env-file .env down`. Намеренное удаление **только синтетических локальных** данных: та же команда с `-v`. Не запускать `-v` для существующего стенда, который нужно сохранить. Миграции 001–008 применяются под PostgreSQL advisory lock. -## Доказательства критериев +Образы: Python 3.12.12, PostgreSQL 17.6, RabbitMQ 4.1.8, HAProxy 3.0.11, MinIO `RELEASE.2025-10-15T17-29-55Z` (digest), MinIO Client `2025.7.21` (digest); OTel Collector contrib 0.128.0, Prometheus 3.5.0, Loki 3.5.0, Tempo 2.9.5, Grafana 12.0.2. Приложение: FastAPI 0.118.3, OTel SDK/API/OTLP 1.45.0, semantic conventions 0.66b0, psycopg 3.2.12, Pillow 11.3.0, pika 1.4.4. Локальный Colima/arm64: 2 CPU, 2 GiB RAM. На этом хосте Docker build обходил неисправный DNS PyPI через ignored `compose.local.yaml`; обычному окружению файл не нужен. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +## Архитектура и конфигурация + +Собственный стек lab5 перенесён без замены контрактов: HAProxy → две API-реплики → PostgreSQL, приватный MinIO, долговечный RabbitMQ и отдельные publisher/worker/cleanup. API возвращает `202 queued` только после сохранения проверенного staging-объекта и одной транзакции attachment/job/outbox. Publisher подтверждает AMQP-публикацию, worker подтверждает delivery после устойчивого результата. Повторы безопасны по version/lease и уникальности эффектов. Реплика API, воркер или broker могут перезапускаться без потери принятой job; единственные DB/S3/broker/host остаются точками отказа. + +| Роль | Версия, health, данные, пределы | Доступ | +| --- | --- | --- | +| `api ×2` | Python/FastAPI; `/health/ready`; 0,5 CPU/256 MiB на реплику | DB и ограниченный S3 secret; API через LB | +| `lb` | HAProxy 3.0.11; backend health; без тома; 0,5 CPU/128 MiB | `127.0.0.1:8080` | +| `postgres` | 17.6; `pg_isready`; `postgres_data` | только внутренняя сеть, secret | +| `s3` | MinIO, digest; HTTP readiness; `s3_data`; 0,5 CPU/384 MiB | приватный bucket, раздельные app/maintenance secrets | +| `broker` | RabbitMQ 4.1.8; diagnostics ping; `broker_data` | durable quorum queue, AMQP secret | +| `publisher` | Python 3.12.12; DB outbox и broker confirms | DB/AMQP secrets, нет S3 | +| `worker` | Python 3.12.12; serial consumer, 0,5 CPU/768 MiB | DB/AMQP/S3 secrets | +| `staging-cleanup` | Python 3.12.12; периодическая сверка | DB и отдельный maintenance S3 secret | +| `otel-collector` | contrib 0.128.0; internal metrics 8888; 384 MiB | OTLP 4317/4318 только Compose | +| `prometheus` | 3.5.0; `prometheus_data`, retention 7d | `127.0.0.1:9090` | +| `loki` / `tempo` | 3.5.0 / 2.9.5; `loki_data` / `tempo_data`; Tempo 768 MiB, блоки до 5 MB/1 мин | только Compose | +| `grafana` | 12.0.2; `grafana_data`, provisioning из Git | `127.0.0.1:3000`, password secret | + +`.env.example` задаёт локальные порты/пределы; Compose передаёт файлы секретов через `/run/secrets`. OTLP endpoint внутри сети `otel-collector:4318`. Имена `APP_VERSION=lab6`, instance UUID и `deployment.environment.name=lab` записаны как resource attributes. Grafana импортирует три datasource и dashboard из `infra/grafana/`. Prometheus загружает rules из `infra/prometheus/rules.yml`. + +OTel SDK экспортирует асинхронно раз в 5 секунд с 2-секундным timeout и очередями 512 spans/records. Collector применяет memory limiter 256/64 MiB, batch 512/2s и конечные exporter queues: metrics 1000, logs/traces по 64 batches, retry не дольше 30 секунд. Отказ хранилища наблюдаемости не входит в business readiness. Потери сигналов при долгом отказе допустимы и измеряются отдельно. + +## Сигналы и запросы + +| Сигнал | Native Prometheus metric / Loki / Tempo | Единица и labels | +| --- | --- | --- | +| HTTP count, duration | `messenger_http_requests_total`, `messenger_http_duration_seconds_bucket` | запросы, секунды; method/route template/status_class | +| Jobs count, duration | `messenger_jobs_processed_total`, `messenger_jobs_duration_seconds_bucket` | jobs, секунды; job_type/outcome | +| Backlog | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds` | jobs, секунды; queue_name | +| Pool | `messenger_db_pool_used`, `messenger_db_pool_waiting` | connections; service_name из resource | +| Worker | `messenger_worker_busy`, `messenger_worker_capacity` | slots; capacity=1 на serial worker | +| CPU/RAM | `messenger_process_cpu_time_seconds_total`, `messenger_process_rss_bytes` | CPU-секунды, bytes текущего RSS | +| До ready | `messenger_attachments_ready_duration_seconds_bucket` | секунды от durable job creation до result COMMIT | +| Логи/трассы | Loki `{service_name="messenger-api"}` / `messenger-worker`; Tempo trace ID | timestamp, severity, event, trace/span IDs, server/producer/consumer spans | + +Dashboard `infra/grafana/dashboards/telemetry.json` содержит сохранённые запросы и единицы. RPS = `sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m]))`; HTTP 5xx SLI = `sum(rate(...{status_class="5xx"}[1m])) / sum(rate(...[1m]))` для бизнес-запросов. 4xx остаются в знаменателе, отдельно показаны. p50/p95/p99 = `histogram_quantile(q,sum by(le)(rate(*_bucket[5m])))` по всем репликам; средние p95 реплик не используются. Queue age рассчитывается из старейшей queued job в PostgreSQL и не заменяет время обработки. После перезапуска publisher устаревшие remote-write ряды отфильтрованы по sample age <15 с. + +Контекст W3C `traceparent/tracestate` сохраняется в job/outbox и передаётся через брокер; consumer span является потомком producer span. Один upload trace проходит API → publisher → worker → S3/render/DB. Логи API/worker сохраняют `trace_id` и `span_id`; `X-Request-Id` коррелирует HTTP-ошибку. Метки метрик не содержат ID, filename или raw URL. Тела запросов, изображения, токены, пароли, Authorization, signed URLs в telemetry не пишутся. Loki индексирует только service/environment/instance; trace ID хранится как metadata. + +## Проверки критериев + +| Критерий | Команда или источник | Наблюдаемый результат | +| --- | --- | --- | +| Бизнес-контракт с telemetry | `make test`; `student-tests/test_pipeline_faults.py`; `test_publish_confirm_gap.py` | smoke PASS, шесть fault-сценариев PASS, повтор после broker confirm дал один логический ready-эффект | +| Связь трёх сигналов | `student-tests/test_telemetry.py`; Loki и Tempo по одному trace ID | API upload → publisher producer → worker consumer → S3/render/DB; API и worker logs содержат тот же trace ID | +| Метрики и dashboard | Grafana API `GET /api/dashboards/uid/lab6-telemetry`; каждый PromQL из dashboard через `/api/v1/query` | dashboard импортирован, 16 панелей; запросы к реально существующим рядам возвращают `success`; inactive alert не создаёт ряд firing | +| Alert и worker stop | `student-tests/test_queue_incident.py` | 8/8 accepted→ready, queue age peak 34,47 с, `Lab6QueueLag` firing через 35 с, inactive через 12 с после запуска worker; `evidence/queue-worker-stop-2026-10-02.json` | +| Alert и DB outage | `student-tests/test_slo_alerts.py --case short/long` | 12/46 ответов 503; короткое и длинное API burn rules прошли firing→inactive; `evidence/slo-short-db-2026-10-02.json`, `slo-long-db-2026-10-02.json` | +| Отказ telemetry stores | `student-tests/test_observability_failure.py` | остановлены Prometheus/Loki/Tempo; smoke PASS, 17 876 POST без ошибки, 595,531 RPS; Collector RSS 70,82→116 MiB при лимите 384 MiB; все stores `up=1` непрерывно 30 с после восстановления; `evidence/telemetry-backend-outage-2026-10-02.json` | +| Сохранность и отказ реплики | `make test-persistence ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | существующая сессия и подтверждённые записи сохранились; после остановки одной реплики 113 успешных записей, один неопределённый ответ, первая успешная запись через 3,685 с | +| CI | `.github/workflows/application.yml` | отдельный job создаёт локальные secrets, собирает Compose, ждёт готовность, выполняет `make test` и проверяет rules API; удалённый CI ещё не запускался | + +Публичный `make test` проверяет бизнес-API, но сам по себе не доказывает полноту наблюдаемости. Публичные контракты и тесты не изменялись. Отдельные тесты лежат в `student-tests/`, исходные агрегаты — в `evidence/`. Пять базовых ролей lab5 с fault matrix сохранены, migration 008 добавляет только допустимый trace carrier. + +## SLO и эксперимент трека 5 + +Пороги зафиксированы **до SLO-инъекций** в `evidence/slo-thresholds-2026-10-02.md` и коммитах `5deecdf`, `43bcde7`. Baseline на двух API-репликах: 30,010 с closed-loop, concurrency=4, 18 260/18 260 успешных POST, 608,462 RPS, p50/p95/p99 успешных ответов 4,289/14,966/59,664 мс. Для 40 PNG (20×320×200 и 20×800×600, concurrency=4) — 40/40 ready за 13,029 с; p95 от начала upload до ready 1,305/1,618 с. Raw JSON: `evidence/lab6-http-baseline-2api-2026-10-02.json`, `lab6-image-baseline-2api-2026-10-02.json`. На спокойном 25-секундном наблюдении 6 правил × 6 samples = 36 состояний `inactive`, ложных firing нет (`evidence/slo-quiet-baseline-2026-10-02.jsonl`). Closed-loop тест имеет coordinated omission и клиентские накладные расходы; p95 изображений включает polling по 0,3 с. + +- API SLO: **99%** non-health ответов без 5xx на скользящих 10 мин; 4xx в знаменателе, но не в числителе. В опыте запросы отправлены прямо в живой API-контейнер: HAProxy после потери readiness может сам вернуть 503, а SDK внутри API не видит такие ответы. Это измеряемое ограничение покрытия SLI. +- Ready SLO: **95%** завершённых image jobs дошли до ready за ≤5 с от durable job creation на скользящих 10 мин. Время upload до `202` исключено; ещё не завершённая job не входит в histogram, поэтому отдельный queue age alert обязателен. +- Burn-rate: short сочетает окна 15/30 с, >14,4× бюджета, `for:10s`; long — 60/120 с, >6×, `for:90s`. Для API error ratio это >14,4% и >6%; для ready slow ratio >72% и >30%. Оценка каждые 5 с. Формулы с фактическими buckets `le="5"` находятся в `infra/prometheus/rules.yml`; `promtool check rules` нашёл 6 правил. Время обнаружения считается от команды воздействия до первого сохранённого `firing`, восстановление — от healthy зависимости до первого `inactive`. Дискретизация 5 с даёт погрешность ±5 с, плюс задержка экспорта. + +| Опыт | Воздействие и выборка | Обнаружение | Восстановление и ложные срабатывания | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Короткий DB | остановка PostgreSQL на 50 с; 12 × 503, 243 × 401 в фиктивном auth-probe | HTTP error и API short firing через 35 с | API short inactive через 22 с после DB healthy; HTTP error через 37 с; API long остался pending и затем inactive, firing не было | +| Длинный DB | остановка PostgreSQL на 180 с; 46 × 503, 234 × 401 | HTTP error 30 с; API short 36 с; API long 141 с | после DB healthy: short 68 с, long 98 с, HTTP error 78 с до inactive; данных не потеряно | +| Worker stop | 8 real uploads при остановленном worker | queue lag firing через 35 с, возраст достиг 34,47 с | 8/8 ready; queue lag inactive через 12 с после запуска; ready histogram был слеп до завершения jobs | +| Умеренный CPU throttle | worker 0,1 CPU, 40 image jobs | ready p95 больших PNG 4,757 с, peak slow ratio 11,36% | пороги burn-rate не достигнуты; 30/30 samples ready short inactive | +| Сильный CPU throttle | worker 0,05 CPU, 30 больших PNG | 30/30 ready за 72,988 с, p95 10,360 с, peak slow ratio 100%; ready short firing с 13:36:26 UTC | лимит возвращён 0,5 CPU; short/long в конце окна inactive; long дал задержанное firing после завершения нагрузки из-за 120-секундной памяти окна и `for:90s` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Пробник с четырьмя одновременными клиентами в коротком DB-опыте сам создавал клиентские timeout вместо устойчивого ряда 503; перед финальными сериями его сменили на один равномерный поток после отдельной проверки `503` через живой API. Пороги не меняли. Файлы финальных серий содержат UTC timestamps, все состояния rules и обезличенные коды ответов. Сильный CPU throttle показал известный компромисс длинного окна: сигнал может сработать уже после прекращения воздействия, пока бюджет ещё расходуется по историческим завершениям. Это одно запаздывающее оповещение, а не новая ошибка после восстановления. -## Эксперименты и отказы +## Отказ наблюдаемости и сохранённые сигналы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +В финальном опыте с остановкой всех трёх storage public smoke прошёл, 30-секундная нагрузка дала 17 876 POST без ошибок, 595,531 RPS, latency успехов p50/p95/p99 4,844/16,108/49,174 мс. Collector имел RSS 70,82 MiB до, 104,2 MiB во время, 116 MiB через 35 с; контейнерный предел 384 MiB. Дельта Collector counters: `otelcol_exporter_enqueue_failed_spans` +19 486, `send_failed_spans` +16 547, `send_failed_log_records` +11 063, `send_failed_metric_points` +484; receiver refused counters остались 0. `send_failed_*` считают неудачные попытки экспорта и не равны потерянным элементам; `enqueue_failed_spans` означает отказ поставить spans в конечную очередь. API не блокировался, но часть traces потеряна. При пропаже telemetry отсутствующая точка не считается нулём ошибок. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +В предварительном опыте `compose up` вернул код 0, но Tempo завершился по OOM при flush накопленного блока 14 MiB; первоначальная проверка восстановления была недостаточной. WAL этого синтетического блока сохранён в локальной копии `/private/tmp/lab6-tempo-wal-backup-2026-10-02`, проблемный блок изолирован внутри тома, остальные сохранённые traces доступны. Tempo получил лимит 768 MiB и блоки до 5 MB/1 мин с одним одновременным flush. Повторная нагрузка с работающей telemetry дала 18 860/18 860 успешных POST, 628,445 RPS, и Tempo оставался `up=1` после flush (`evidence/tempo-recovery-load-2026-10-02.json`). Финальный отказ хранилищ после исправления прошёл; тест восстановления требует 30 секунд непрерывного `up=1` у Prometheus, Loki и Tempo. -## Ограничения и следующие шаги +Grafana импортировала обновлённый dashboard через provisioning; Prometheus сохранил rule states, Loki — структурные API/worker logs, Tempo — один trace с API upload, publisher, worker и дочерними S3/render/DB spans. Пример доказанной цепочки в локальном хранилище: trace ID `5626970bc7d05d13fb839033d5f9b410`, worker consumer span `8850acdc538bf1ae`, API root `0ee38f12aafbf091`, publisher producer `9ef172904a867da6`. Trace ID — случайный технический идентификатор синтетического запроса; тела, credentials и URL не сохранены в evidence. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Пробный слепой разбор runbook выполнил отдельный агент только по сохранённым Prometheus/Loki/Tempo сигналам, без сценариев инъекций и Docker stdout. Для окна 13:16:20–13:21:15 UTC он нашёл первые 5xx в 13:16:30, рост ожидания DB pool до 418 и HTTP p95 до 9,75 с; предположил деградацию БД/пула, но не мог различить их без DB child span. Для 13:23:45–13:24:50 UTC он увидел восемь `job_published`, queue depth 8, рост возраста до 34,5 с, отсутствие свежей worker capacity и восемь `job_processed` после восстановления; предположил недоступность worker. Во время разбора `up{job="tempo"}=0` из-за описанного OOM, поэтому trace не был доступен, а причина остановки worker не доказывалась одними сохранёнными сигналами. Это проверка понятности runbook другим агентом, не независимая проверка студентом. + +Runbook: `infra/runbooks/incident.md`. При пропаже сигнала оператор сначала проверяет `up` storage и счётчики Collector. Локальный стенд не гарантирует доступности при потере одного PostgreSQL/S3/RabbitMQ/host, а serial worker ограничивает throughput изображений. Синтетические тесты не доказывают production SLO или безошибочность внешнего клиента. diff --git a/compose.telemetry.yaml b/compose.telemetry.yaml new file mode 100644 index 0000000..2513324 --- /dev/null +++ b/compose.telemetry.yaml @@ -0,0 +1,68 @@ +# Add with: docker compose -f compose.yaml -f compose.telemetry.yaml up -d +services: + otel-collector: + image: otel/opentelemetry-collector-contrib:0.128.0 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + mem_limit: 384m + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + depends_on: + - prometheus + - loki + - tempo + + prometheus: + image: prom/prometheus:v3.5.0 + ports: + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --web.enable-remote-write-receiver + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules.yml:/etc/prometheus/rules.yml:ro + - prometheus_data:/prometheus + + loki: + image: grafana/loki:3.5.0 + command: ["-config.file=/etc/loki/config.yaml"] + volumes: + - ./infra/loki/config.yaml:/etc/loki/config.yaml:ro + - loki_data:/loki + + tempo: + image: grafana/tempo:2.9.5 + mem_limit: 768m + command: ["-config.file=/etc/tempo/config.yaml"] + volumes: + - ./infra/tempo/config.yaml:/etc/tempo/config.yaml:ro + - tempo_data:/var/tempo + + grafana: + image: grafana/grafana:12.0.2 + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + environment: + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + secrets: [grafana_admin_password] + volumes: + - grafana_data:/var/lib/grafana + - ./infra/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro + - ./infra/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + - loki + - tempo + +volumes: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + +secrets: + grafana_admin_password: + file: ./secrets/grafana_admin_password diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..345da2d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,231 @@ +name: mtusi2026lab6 + +services: + postgres: + image: postgres:17.6-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + + migrate: + build: . + command: ["python", "scripts/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + restart: "no" + + s3-volume-init: + image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4 + user: "0:0" + entrypoint: ["/bin/sh", "-c"] + command: ["chown -R 100:100 /data"] + volumes: [s3_data:/data] + restart: "no" + + s3: + image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4 + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + secrets: [minio_root_user, minio_root_password] + volumes: [s3_data:/data] + depends_on: + s3-volume-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9000/minio/health/ready"] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 384m + + s3-bootstrap: + image: bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158 + user: "0:0" + entrypoint: ["/bin/sh", "/bootstrap.sh"] + command: [] + volumes: + - ./infra/bootstrap-s3.sh:/bootstrap.sh:ro + - ./infra/minio-api-policy.json:/policies/minio-api-policy.json:ro + - ./infra/minio-maintenance-policy.json:/policies/minio-maintenance-policy.json:ro + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + - s3_maintenance_key_id + - s3_maintenance_secret_key + depends_on: + s3: + condition: service_healthy + restart: "no" + + broker: + image: rabbitmq:4.1.8-management + hostname: broker + restart: unless-stopped + volumes: + - broker_data:/var/lib/rabbitmq + - ./secrets/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 24 + + broker-init: + image: python:3.12.14-alpine3.24 + restart: "no" + depends_on: + broker: + condition: service_healthy + environment: + BROKER_URL_FILE: /run/secrets/broker_url + volumes: + - ./scripts/bootstrap_broker.py:/app/bootstrap_broker.py:ro + secrets: [broker_url] + command: ["python", "/app/bootstrap_broker.py"] + + api: + build: . + scale: 2 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + APP_VERSION: lab6 + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 5s + stop_grace_period: 10s + cpus: 0.5 + mem_limit: 256m + + publisher: + build: . + command: ["python", "-m", "src.publisher"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + BROKER_QUEUE: attachments.process + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + APP_VERSION: lab6 + secrets: [database_url, broker_url] + depends_on: + migrate: + condition: service_completed_successfully + broker-init: + condition: service_completed_successfully + restart: unless-stopped + + staging-cleanup: + build: . + command: ["python", "-m", "src.staging_cleanup"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: s3:9000 + S3_MAINTENANCE_KEY_ID_FILE: /run/secrets/s3_maintenance_key_id + S3_MAINTENANCE_SECRET_KEY_FILE: /run/secrets/s3_maintenance_secret_key + STAGING_GRACE_SECONDS: ${STAGING_GRACE_SECONDS:-3600} + STAGING_CLEANUP_INTERVAL_SECONDS: ${STAGING_CLEANUP_INTERVAL_SECONDS:-60} + secrets: [database_url, s3_maintenance_key_id, s3_maintenance_secret_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + restart: unless-stopped + + worker: + build: . + command: ["python", "-m", "src.worker"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + BROKER_QUEUE: attachments.process + S3_ENDPOINT: s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + APP_VERSION: lab6 + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + broker-init: + condition: service_completed_successfully + restart: unless-stopped + cpus: 0.5 + mem_limit: 768m + + lb: + image: haproxy:3.0.11-alpine + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + api: + condition: service_healthy + cpus: 0.5 + mem_limit: 128m + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_maintenance_key_id: + file: ./secrets/s3_maintenance_key_id + s3_maintenance_secret_key: + file: ./secrets/s3_maintenance_secret_key + broker_url: + file: ./secrets/broker_url + +volumes: + postgres_data: + s3_data: + broker_data: diff --git a/evidence/lab6-http-baseline-2api-2026-10-02.json b/evidence/lab6-http-baseline-2api-2026-10-02.json new file mode 100644 index 0000000..0751892 --- /dev/null +++ b/evidence/lab6-http-baseline-2api-2026-10-02.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.01, + "concurrency": 4, + "successful_requests": 18260, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 608.462, + "successful_latency_ms": { + "p50": 4.289, + "p95": 14.966, + "p99": 59.664 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-image-baseline-2api-2026-10-02.json b/evidence/lab6-image-baseline-2api-2026-10-02.json new file mode 100644 index 0000000..408b1c7 --- /dev/null +++ b/evidence/lab6-image-baseline-2api-2026-10-02.json @@ -0,0 +1,40 @@ +{ + "accepted_ready": 40, + "concurrency": 4, + "elapsed_seconds": 13.029, + "failed": 0, + "failure_types": [], + "lab": 6, + "per_size": 20, + "profiles": { + "noise_800x600": { + "ready_ms": { + "p50": 1309.41, + "p95": 1618.28, + "p99": 1645.672 + }, + "size_bytes": 1441103, + "successful": 20, + "upload_ms": { + "p50": 34.166, + "p95": 52.97, + "p99": 84.31 + } + }, + "small_320x200": { + "ready_ms": { + "p50": 988.508, + "p95": 1304.924, + "p99": 1332.116 + }, + "size_bytes": 111817, + "successful": 20, + "upload_ms": { + "p50": 24.969, + "p95": 31.176, + "p99": 33.817 + } + } + }, + "workload": "synthetic PNG upload and poll until ready" +} diff --git a/evidence/queue-worker-stop-2026-10-02.json b/evidence/queue-worker-stop-2026-10-02.json new file mode 100644 index 0000000..0abbac5 --- /dev/null +++ b/evidence/queue-worker-stop-2026-10-02.json @@ -0,0 +1,324 @@ +{ + "prometheus_samples": [ + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:44+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:49+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:54+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:59+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:04+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 4.44206, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:09+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 9.447189, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:14+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 14.45216, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "pending", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:19+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 19.454665, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "pending", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:24+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 24.459164, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "firing", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:29+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 29.464961, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "firing", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:34+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 34.474193, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "firing", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:39+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:44+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:49+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:54+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:59+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:25:04+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:25:09+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + } + ], + "queue_alert_states": [ + "firing", + "inactive", + "pending" + ], + "queued_uploads": 8, + "ready_uploads": 8, + "times": { + "all_ready": "2026-10-02T13:24:33+00:00", + "started": "2026-10-02T13:23:44+00:00", + "uploads_accepted": "2026-10-02T13:23:54+00:00", + "worker_started": "2026-10-02T13:24:32+00:00", + "worker_stopped": "2026-10-02T13:23:54+00:00" + } +} diff --git a/evidence/ready-throttle-load-2026-10-02.json b/evidence/ready-throttle-load-2026-10-02.json new file mode 100644 index 0000000..c63deac --- /dev/null +++ b/evidence/ready-throttle-load-2026-10-02.json @@ -0,0 +1,40 @@ +{ + "accepted_ready": 40, + "concurrency": 4, + "elapsed_seconds": 29.794, + "failed": 0, + "failure_types": [], + "lab": 6, + "per_size": 20, + "profiles": { + "noise_800x600": { + "ready_ms": { + "p50": 4105.489, + "p95": 4756.876, + "p99": 5080.984 + }, + "size_bytes": 1441103, + "successful": 20, + "upload_ms": { + "p50": 29.131, + "p95": 52.165, + "p99": 53.056 + } + }, + "small_320x200": { + "ready_ms": { + "p50": 1573.785, + "p95": 1916.581, + "p99": 1919.551 + }, + "size_bytes": 111817, + "successful": 20, + "upload_ms": { + "p50": 17.66, + "p95": 27.092, + "p99": 66.386 + } + } + }, + "workload": "synthetic PNG upload and poll until ready" +} diff --git a/evidence/ready-throttle-strong-load-2026-10-02.json b/evidence/ready-throttle-strong-load-2026-10-02.json new file mode 100644 index 0000000..6cba4bd --- /dev/null +++ b/evidence/ready-throttle-strong-load-2026-10-02.json @@ -0,0 +1,26 @@ +{ + "accepted_ready": 30, + "concurrency": 4, + "elapsed_seconds": 72.988, + "failed": 0, + "failure_types": [], + "lab": 6, + "per_size": 30, + "profiles": { + "noise_800x600": { + "ready_ms": { + "p50": 9476.253, + "p95": 10359.763, + "p99": 10437.545 + }, + "size_bytes": 1441103, + "successful": 30, + "upload_ms": { + "p50": 27.409, + "p95": 68.138, + "p99": 83.338 + } + } + }, + "workload": "synthetic PNG upload and poll until ready" +} diff --git a/evidence/ready-throttle-strong-watch-2026-10-02.jsonl b/evidence/ready-throttle-strong-watch-2026-10-02.jsonl new file mode 100644 index 0000000..23f8fa9 --- /dev/null +++ b/evidence/ready-throttle-strong-watch-2026-10-02.jsonl @@ -0,0 +1,32 @@ +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:31+00:00", "values": {"collector_failed_spans": 20872.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:36+00:00", "values": {"collector_failed_spans": 20878.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:41+00:00", "values": {"collector_failed_spans": 20900.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:46+00:00", "values": {"collector_failed_spans": 20912.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:51+00:00", "values": {"collector_failed_spans": 20918.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:56+00:00", "values": {"collector_failed_spans": 20930.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:01+00:00", "values": {"collector_failed_spans": 20930.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.699382, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:06+00:00", "values": {"collector_failed_spans": 21312.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.699382, "ready_slow_ratio_30s": 0.6666666666666666}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:11+00:00", "values": {"collector_failed_spans": 21914.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.303315, "ready_slow_ratio_30s": 0.8333333333333333}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "pending"}, "timestamp": "2026-10-02T13:36:16+00:00", "values": {"collector_failed_spans": 22138.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.303315, "ready_slow_ratio_30s": 0.875}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "pending"}, "timestamp": "2026-10-02T13:36:21+00:00", "values": {"collector_failed_spans": 22264.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.819518, "ready_slow_ratio_30s": 0.9}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:26+00:00", "values": {"collector_failed_spans": 22264.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.819518, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:31+00:00", "values": {"collector_failed_spans": 22752.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.800324, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:36+00:00", "values": {"collector_failed_spans": 22807.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.800324, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:41+00:00", "values": {"collector_failed_spans": 22807.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 6.705292, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:46+00:00", "values": {"collector_failed_spans": 22817.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 6.705292, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:51+00:00", "values": {"collector_failed_spans": 22846.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.390955, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:56+00:00", "values": {"collector_failed_spans": 22852.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.390955, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:01+00:00", "values": {"collector_failed_spans": 22870.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.015189, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:06+00:00", "values": {"collector_failed_spans": 22876.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.015189, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:11+00:00", "values": {"collector_failed_spans": 22888.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:16+00:00", "values": {"collector_failed_spans": 22899.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:21+00:00", "values": {"collector_failed_spans": 22917.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:26+00:00", "values": {"collector_failed_spans": 22929.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:31+00:00", "values": {"collector_failed_spans": 22939.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:36+00:00", "values": {"collector_failed_spans": 22951.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:41+00:00", "values": {"collector_failed_spans": 22963.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:46+00:00", "values": {"collector_failed_spans": 22975.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:51+00:00", "values": {"collector_failed_spans": 22987.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:56+00:00", "values": {"collector_failed_spans": 22993.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:38:01+00:00", "values": {"collector_failed_spans": 23016.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:38:06+00:00", "values": {"collector_failed_spans": 23022.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} diff --git a/evidence/ready-throttle-watch-2026-10-02.jsonl b/evidence/ready-throttle-watch-2026-10-02.jsonl new file mode 100644 index 0000000..9ac20cb --- /dev/null +++ b/evidence/ready-throttle-watch-2026-10-02.jsonl @@ -0,0 +1,30 @@ +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:32:53+00:00", "values": {"collector_failed_spans": 20481.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:32:58+00:00", "values": {"collector_failed_spans": 20491.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:03+00:00", "values": {"collector_failed_spans": 20509.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:08+00:00", "values": {"collector_failed_spans": 20527.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:13+00:00", "values": {"collector_failed_spans": 20531.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:18+00:00", "values": {"collector_failed_spans": 20537.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:23+00:00", "values": {"collector_failed_spans": 20549.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:28+00:00", "values": {"collector_failed_spans": 20561.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:33+00:00", "values": {"collector_failed_spans": 20585.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.155266, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:38+00:00", "values": {"collector_failed_spans": 20591.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.998314, "ready_slow_ratio_30s": 0.05458974358974353}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:43+00:00", "values": {"collector_failed_spans": 20609.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.081209, "ready_slow_ratio_30s": 0.09646666666666656}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:48+00:00", "values": {"collector_failed_spans": 20627.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.683843, "ready_slow_ratio_30s": 0.1055743589743589}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:53+00:00", "values": {"collector_failed_spans": 20637.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.220981, "ready_slow_ratio_30s": 0.11358709677419333}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:58+00:00", "values": {"collector_failed_spans": 20643.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.589964, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:03+00:00", "values": {"collector_failed_spans": 20655.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:08+00:00", "values": {"collector_failed_spans": 20661.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:14+00:00", "values": {"collector_failed_spans": 20679.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:19+00:00", "values": {"collector_failed_spans": 20701.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:24+00:00", "values": {"collector_failed_spans": 20707.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:29+00:00", "values": {"collector_failed_spans": 20719.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:34+00:00", "values": {"collector_failed_spans": 20731.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:39+00:00", "values": {"collector_failed_spans": 20743.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:44+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:49+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:54+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:59+00:00", "values": {"collector_failed_spans": 20797.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:04+00:00", "values": {"collector_failed_spans": 20803.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:09+00:00", "values": {"collector_failed_spans": 20821.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:14+00:00", "values": {"collector_failed_spans": 20838.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:19+00:00", "values": {"collector_failed_spans": 20848.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} diff --git a/evidence/slo-long-db-2026-10-02.json b/evidence/slo-long-db-2026-10-02.json new file mode 100644 index 0000000..cc27859 --- /dev/null +++ b/evidence/slo-long-db-2026-10-02.json @@ -0,0 +1,2262 @@ +{ + "case": "long", + "events": [ + [ + "2026-10-02T13:16:15+00:00", + 401 + ], + [ + "2026-10-02T13:16:15+00:00", + 401 + ], + [ + "2026-10-02T13:16:16+00:00", + 401 + ], + [ + "2026-10-02T13:16:16+00:00", + 401 + ], + [ + "2026-10-02T13:16:16+00:00", + 401 + ], + [ + "2026-10-02T13:16:17+00:00", + 401 + ], + [ + "2026-10-02T13:16:17+00:00", + 401 + ], + [ + "2026-10-02T13:16:18+00:00", + 401 + ], + [ + "2026-10-02T13:16:18+00:00", + 401 + ], + [ + "2026-10-02T13:16:18+00:00", + 401 + ], + [ + "2026-10-02T13:16:19+00:00", + 401 + ], + [ + "2026-10-02T13:16:19+00:00", + 401 + ], + [ + "2026-10-02T13:16:20+00:00", + 401 + ], + [ + "2026-10-02T13:16:20+00:00", + 401 + ], + [ + "2026-10-02T13:16:21+00:00", + 401 + ], + [ + "2026-10-02T13:16:21+00:00", + 401 + ], + [ + "2026-10-02T13:16:21+00:00", + 401 + ], + [ + "2026-10-02T13:16:22+00:00", + 401 + ], + [ + "2026-10-02T13:16:22+00:00", + 401 + ], + [ + "2026-10-02T13:16:23+00:00", + 401 + ], + [ + "2026-10-02T13:16:23+00:00", + 401 + ], + [ + "2026-10-02T13:16:23+00:00", + 401 + ], + [ + "2026-10-02T13:16:24+00:00", + 401 + ], + [ + "2026-10-02T13:16:24+00:00", + 401 + ], + [ + "2026-10-02T13:16:25+00:00", + 401 + ], + [ + "2026-10-02T13:16:25+00:00", + 401 + ], + [ + "2026-10-02T13:16:25+00:00", + 401 + ], + [ + "2026-10-02T13:16:26+00:00", + 401 + ], + [ + "2026-10-02T13:16:26+00:00", + 401 + ], + [ + "2026-10-02T13:16:27+00:00", + 401 + ], + [ + "2026-10-02T13:16:27+00:00", + 401 + ], + [ + "2026-10-02T13:16:27+00:00", + 401 + ], + [ + "2026-10-02T13:16:28+00:00", + 401 + ], + [ + "2026-10-02T13:16:28+00:00", + 401 + ], + [ + "2026-10-02T13:16:29+00:00", + 401 + ], + [ + "2026-10-02T13:16:29+00:00", + 401 + ], + [ + "2026-10-02T13:16:30+00:00", + 503 + ], + [ + "2026-10-02T13:16:30+00:00", + 503 + ], + [ + "2026-10-02T13:16:35+00:00", + 503 + ], + [ + "2026-10-02T13:16:41+00:00", + 503 + ], + [ + "2026-10-02T13:16:46+00:00", + 503 + ], + [ + "2026-10-02T13:16:52+00:00", + 503 + ], + [ + "2026-10-02T13:16:57+00:00", + 503 + ], + [ + "2026-10-02T13:17:02+00:00", + 503 + ], + [ + "2026-10-02T13:17:08+00:00", + 503 + ], + [ + "2026-10-02T13:17:13+00:00", + 503 + ], + [ + "2026-10-02T13:17:19+00:00", + 503 + ], + [ + "2026-10-02T13:17:24+00:00", + 503 + ], + [ + "2026-10-02T13:17:29+00:00", + 503 + ], + [ + "2026-10-02T13:17:35+00:00", + 503 + ], + [ + "2026-10-02T13:17:40+00:00", + 503 + ], + [ + "2026-10-02T13:17:46+00:00", + 503 + ], + [ + "2026-10-02T13:17:51+00:00", + 503 + ], + [ + "2026-10-02T13:17:57+00:00", + 503 + ], + [ + "2026-10-02T13:18:02+00:00", + 503 + ], + [ + "2026-10-02T13:18:07+00:00", + 503 + ], + [ + "2026-10-02T13:18:13+00:00", + 503 + ], + [ + "2026-10-02T13:18:18+00:00", + 503 + ], + [ + "2026-10-02T13:18:24+00:00", + 503 + ], + [ + "2026-10-02T13:18:29+00:00", + 503 + ], + [ + "2026-10-02T13:18:34+00:00", + 503 + ], + [ + "2026-10-02T13:18:40+00:00", + 503 + ], + [ + "2026-10-02T13:18:45+00:00", + 503 + ], + [ + "2026-10-02T13:18:51+00:00", + 503 + ], + [ + "2026-10-02T13:18:56+00:00", + 503 + ], + [ + "2026-10-02T13:19:01+00:00", + 503 + ], + [ + "2026-10-02T13:19:07+00:00", + 503 + ], + [ + "2026-10-02T13:19:12+00:00", + 503 + ], + [ + "2026-10-02T13:19:18+00:00", + 503 + ], + [ + "2026-10-02T13:19:23+00:00", + 503 + ], + [ + "2026-10-02T13:19:28+00:00", + 503 + ], + [ + "2026-10-02T13:19:34+00:00", + 503 + ], + [ + "2026-10-02T13:19:39+00:00", + 503 + ], + [ + "2026-10-02T13:19:45+00:00", + 503 + ], + [ + "2026-10-02T13:19:50+00:00", + 503 + ], + [ + "2026-10-02T13:19:55+00:00", + 503 + ], + [ + "2026-10-02T13:20:01+00:00", + 503 + ], + [ + "2026-10-02T13:20:06+00:00", + 503 + ], + [ + "2026-10-02T13:20:12+00:00", + 503 + ], + [ + "2026-10-02T13:20:17+00:00", + 503 + ], + [ + "2026-10-02T13:20:23+00:00", + 503 + ], + [ + "2026-10-02T13:20:28+00:00", + 503 + ], + [ + "2026-10-02T13:20:29+00:00", + 401 + ], + [ + "2026-10-02T13:20:30+00:00", + 401 + ], + [ + "2026-10-02T13:20:30+00:00", + 401 + ], + [ + "2026-10-02T13:20:30+00:00", + 401 + ], + [ + "2026-10-02T13:20:31+00:00", + 401 + ], + [ + "2026-10-02T13:20:31+00:00", + 401 + ], + [ + "2026-10-02T13:20:32+00:00", + 401 + ], + [ + "2026-10-02T13:20:32+00:00", + 401 + ], + [ + "2026-10-02T13:20:33+00:00", + 401 + ], + [ + "2026-10-02T13:20:33+00:00", + 401 + ], + [ + "2026-10-02T13:20:33+00:00", + 401 + ], + [ + "2026-10-02T13:20:34+00:00", + 401 + ], + [ + "2026-10-02T13:20:34+00:00", + 401 + ], + [ + "2026-10-02T13:20:35+00:00", + 401 + ], + [ + "2026-10-02T13:20:35+00:00", + 401 + ], + [ + "2026-10-02T13:20:35+00:00", + 401 + ], + [ + "2026-10-02T13:20:36+00:00", + 401 + ], + [ + "2026-10-02T13:20:36+00:00", + 401 + ], + [ + "2026-10-02T13:20:37+00:00", + 401 + ], + [ + "2026-10-02T13:20:37+00:00", + 401 + ], + [ + "2026-10-02T13:20:37+00:00", + 401 + ], + [ + "2026-10-02T13:20:38+00:00", + 401 + ], + [ + "2026-10-02T13:20:38+00:00", + 401 + ], + [ + "2026-10-02T13:20:39+00:00", + 401 + ], + [ + "2026-10-02T13:20:39+00:00", + 401 + ], + [ + "2026-10-02T13:20:39+00:00", + 401 + ], + [ + "2026-10-02T13:20:40+00:00", + 401 + ], + [ + "2026-10-02T13:20:40+00:00", + 401 + ], + [ + "2026-10-02T13:20:41+00:00", + 401 + ], + [ + "2026-10-02T13:20:41+00:00", + 401 + ], + [ + "2026-10-02T13:20:42+00:00", + 401 + ], + [ + "2026-10-02T13:20:42+00:00", + 401 + ], + [ + "2026-10-02T13:20:42+00:00", + 401 + ], + [ + "2026-10-02T13:20:43+00:00", + 401 + ], + [ + "2026-10-02T13:20:43+00:00", + 401 + ], + [ + "2026-10-02T13:20:44+00:00", + 401 + ], + [ + "2026-10-02T13:20:44+00:00", + 401 + ], + [ + "2026-10-02T13:20:44+00:00", + 401 + ], + [ + "2026-10-02T13:20:45+00:00", + 401 + ], + [ + "2026-10-02T13:20:45+00:00", + 401 + ], + [ + "2026-10-02T13:20:46+00:00", + 401 + ], + [ + "2026-10-02T13:20:46+00:00", + 401 + ], + [ + "2026-10-02T13:20:46+00:00", + 401 + ], + [ + "2026-10-02T13:20:47+00:00", + 401 + ], + [ + "2026-10-02T13:20:47+00:00", + 401 + ], + [ + "2026-10-02T13:20:48+00:00", + 401 + ], + [ + "2026-10-02T13:20:48+00:00", + 401 + ], + [ + "2026-10-02T13:20:48+00:00", + 401 + ], + [ + "2026-10-02T13:20:49+00:00", + 401 + ], + [ + "2026-10-02T13:20:49+00:00", + 401 + ], + [ + "2026-10-02T13:20:50+00:00", + 401 + ], + [ + "2026-10-02T13:20:50+00:00", + 401 + ], + [ + "2026-10-02T13:20:50+00:00", + 401 + ], + [ + "2026-10-02T13:20:51+00:00", + 401 + ], + [ + "2026-10-02T13:20:51+00:00", + 401 + ], + [ + "2026-10-02T13:20:52+00:00", + 401 + ], + [ + "2026-10-02T13:20:52+00:00", + 401 + ], + [ + "2026-10-02T13:20:53+00:00", + 401 + ], + [ + "2026-10-02T13:20:53+00:00", + 401 + ], + [ + "2026-10-02T13:20:53+00:00", + 401 + ], + [ + "2026-10-02T13:20:54+00:00", + 401 + ], + [ + "2026-10-02T13:20:54+00:00", + 401 + ], + [ + "2026-10-02T13:20:55+00:00", + 401 + ], + [ + "2026-10-02T13:20:55+00:00", + 401 + ], + [ + "2026-10-02T13:20:55+00:00", + 401 + ], + [ + "2026-10-02T13:20:56+00:00", + 401 + ], + [ + "2026-10-02T13:20:56+00:00", + 401 + ], + [ + "2026-10-02T13:20:57+00:00", + 401 + ], + [ + "2026-10-02T13:20:57+00:00", + 401 + ], + [ + "2026-10-02T13:20:57+00:00", + 401 + ], + [ + "2026-10-02T13:20:58+00:00", + 401 + ], + [ + "2026-10-02T13:20:58+00:00", + 401 + ], + [ + "2026-10-02T13:20:59+00:00", + 401 + ], + [ + "2026-10-02T13:20:59+00:00", + 401 + ], + [ + "2026-10-02T13:20:59+00:00", + 401 + ], + [ + "2026-10-02T13:21:00+00:00", + 401 + ], + [ + "2026-10-02T13:21:00+00:00", + 401 + ], + [ + "2026-10-02T13:21:01+00:00", + 401 + ], + [ + "2026-10-02T13:21:01+00:00", + 401 + ], + [ + "2026-10-02T13:21:01+00:00", + 401 + ], + [ + "2026-10-02T13:21:02+00:00", + 401 + ], + [ + "2026-10-02T13:21:02+00:00", + 401 + ], + [ + "2026-10-02T13:21:03+00:00", + 401 + ], + [ + "2026-10-02T13:21:03+00:00", + 401 + ], + [ + "2026-10-02T13:21:04+00:00", + 401 + ], + [ + "2026-10-02T13:21:04+00:00", + 401 + ], + [ + "2026-10-02T13:21:04+00:00", + 401 + ], + [ + "2026-10-02T13:21:05+00:00", + 401 + ], + [ + "2026-10-02T13:21:05+00:00", + 401 + ], + [ + "2026-10-02T13:21:06+00:00", + 401 + ], + [ + "2026-10-02T13:21:06+00:00", + 401 + ], + [ + "2026-10-02T13:21:06+00:00", + 401 + ], + [ + "2026-10-02T13:21:07+00:00", + 401 + ], + [ + "2026-10-02T13:21:07+00:00", + 401 + ], + [ + "2026-10-02T13:21:08+00:00", + 401 + ], + [ + "2026-10-02T13:21:08+00:00", + 401 + ], + [ + "2026-10-02T13:21:08+00:00", + 401 + ], + [ + "2026-10-02T13:21:09+00:00", + 401 + ], + [ + "2026-10-02T13:21:09+00:00", + 401 + ], + [ + "2026-10-02T13:21:10+00:00", + 401 + ], + [ + "2026-10-02T13:21:10+00:00", + 401 + ], + [ + "2026-10-02T13:21:10+00:00", + 401 + ], + [ + "2026-10-02T13:21:11+00:00", + 401 + ], + [ + "2026-10-02T13:21:11+00:00", + 401 + ], + [ + "2026-10-02T13:21:12+00:00", + 401 + ], + [ + "2026-10-02T13:21:12+00:00", + 401 + ], + [ + "2026-10-02T13:21:12+00:00", + 401 + ], + [ + "2026-10-02T13:21:13+00:00", + 401 + ], + [ + "2026-10-02T13:21:13+00:00", + 401 + ], + [ + "2026-10-02T13:21:14+00:00", + 401 + ], + [ + "2026-10-02T13:21:14+00:00", + 401 + ], + [ + "2026-10-02T13:21:15+00:00", + 401 + ], + [ + "2026-10-02T13:21:15+00:00", + 401 + ], + [ + "2026-10-02T13:21:15+00:00", + 401 + ], + [ + "2026-10-02T13:21:16+00:00", + 401 + ], + [ + "2026-10-02T13:21:16+00:00", + 401 + ], + [ + "2026-10-02T13:21:17+00:00", + 401 + ], + [ + "2026-10-02T13:21:17+00:00", + 401 + ], + [ + "2026-10-02T13:21:17+00:00", + 401 + ], + [ + "2026-10-02T13:21:18+00:00", + 401 + ], + [ + "2026-10-02T13:21:18+00:00", + 401 + ], + [ + "2026-10-02T13:21:19+00:00", + 401 + ], + [ + "2026-10-02T13:21:19+00:00", + 401 + ], + [ + "2026-10-02T13:21:19+00:00", + 401 + ], + [ + "2026-10-02T13:21:20+00:00", + 401 + ], + [ + "2026-10-02T13:21:20+00:00", + 401 + ], + [ + "2026-10-02T13:21:21+00:00", + 401 + ], + [ + "2026-10-02T13:21:21+00:00", + 401 + ], + [ + "2026-10-02T13:21:21+00:00", + 401 + ], + [ + "2026-10-02T13:21:22+00:00", + 401 + ], + [ + "2026-10-02T13:21:22+00:00", + 401 + ], + [ + "2026-10-02T13:21:23+00:00", + 401 + ], + [ + "2026-10-02T13:21:23+00:00", + 401 + ], + [ + "2026-10-02T13:21:23+00:00", + 401 + ], + [ + "2026-10-02T13:21:24+00:00", + 401 + ], + [ + "2026-10-02T13:21:24+00:00", + 401 + ], + [ + "2026-10-02T13:21:25+00:00", + 401 + ], + [ + "2026-10-02T13:21:25+00:00", + 401 + ], + [ + "2026-10-02T13:21:26+00:00", + 401 + ], + [ + "2026-10-02T13:21:26+00:00", + 401 + ], + [ + "2026-10-02T13:21:26+00:00", + 401 + ], + [ + "2026-10-02T13:21:27+00:00", + 401 + ], + [ + "2026-10-02T13:21:27+00:00", + 401 + ], + [ + "2026-10-02T13:21:28+00:00", + 401 + ], + [ + "2026-10-02T13:21:28+00:00", + 401 + ], + [ + "2026-10-02T13:21:28+00:00", + 401 + ], + [ + "2026-10-02T13:21:29+00:00", + 401 + ], + [ + "2026-10-02T13:21:29+00:00", + 401 + ], + [ + "2026-10-02T13:21:30+00:00", + 401 + ], + [ + "2026-10-02T13:21:30+00:00", + 401 + ], + [ + "2026-10-02T13:21:30+00:00", + 401 + ], + [ + "2026-10-02T13:21:31+00:00", + 401 + ], + [ + "2026-10-02T13:21:31+00:00", + 401 + ], + [ + "2026-10-02T13:21:32+00:00", + 401 + ], + [ + "2026-10-02T13:21:32+00:00", + 401 + ], + [ + "2026-10-02T13:21:32+00:00", + 401 + ], + [ + "2026-10-02T13:21:33+00:00", + 401 + ], + [ + "2026-10-02T13:21:33+00:00", + 401 + ], + [ + "2026-10-02T13:21:34+00:00", + 401 + ], + [ + "2026-10-02T13:21:34+00:00", + 401 + ], + [ + "2026-10-02T13:21:34+00:00", + 401 + ], + [ + "2026-10-02T13:21:35+00:00", + 401 + ], + [ + "2026-10-02T13:21:35+00:00", + 401 + ], + [ + "2026-10-02T13:21:36+00:00", + 401 + ], + [ + "2026-10-02T13:21:36+00:00", + 401 + ], + [ + "2026-10-02T13:21:36+00:00", + 401 + ], + [ + "2026-10-02T13:21:37+00:00", + 401 + ], + [ + "2026-10-02T13:21:37+00:00", + 401 + ], + [ + "2026-10-02T13:21:38+00:00", + 401 + ], + [ + "2026-10-02T13:21:38+00:00", + 401 + ], + [ + "2026-10-02T13:21:39+00:00", + 401 + ], + [ + "2026-10-02T13:21:39+00:00", + 401 + ], + [ + "2026-10-02T13:21:39+00:00", + 401 + ], + [ + "2026-10-02T13:21:40+00:00", + 401 + ], + [ + "2026-10-02T13:21:40+00:00", + 401 + ], + [ + "2026-10-02T13:21:41+00:00", + 401 + ], + [ + "2026-10-02T13:21:41+00:00", + 401 + ], + [ + "2026-10-02T13:21:41+00:00", + 401 + ], + [ + "2026-10-02T13:21:42+00:00", + 401 + ], + [ + "2026-10-02T13:21:42+00:00", + 401 + ], + [ + "2026-10-02T13:21:43+00:00", + 401 + ], + [ + "2026-10-02T13:21:43+00:00", + 401 + ], + [ + "2026-10-02T13:21:43+00:00", + 401 + ], + [ + "2026-10-02T13:21:44+00:00", + 401 + ], + [ + "2026-10-02T13:21:44+00:00", + 401 + ], + [ + "2026-10-02T13:21:45+00:00", + 401 + ], + [ + "2026-10-02T13:21:45+00:00", + 401 + ], + [ + "2026-10-02T13:21:45+00:00", + 401 + ], + [ + "2026-10-02T13:21:46+00:00", + 401 + ], + [ + "2026-10-02T13:21:46+00:00", + 401 + ], + [ + "2026-10-02T13:21:47+00:00", + 401 + ], + [ + "2026-10-02T13:21:47+00:00", + 401 + ], + [ + "2026-10-02T13:21:48+00:00", + 401 + ], + [ + "2026-10-02T13:21:48+00:00", + 401 + ], + [ + "2026-10-02T13:21:48+00:00", + 401 + ], + [ + "2026-10-02T13:21:49+00:00", + 401 + ], + [ + "2026-10-02T13:21:49+00:00", + 401 + ], + [ + "2026-10-02T13:21:50+00:00", + 401 + ] + ], + "planned_seconds": { + "outage": 180, + "post": 130, + "pre": 15 + }, + "prometheus_samples": [ + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:14+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:19+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:24+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:29+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:34+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:39+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.05263157894736842, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:44+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.10714285714285715, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:49+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.23529411764705882, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:54+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:59+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.8333333333333333, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.8, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.10714285714285715, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.07407407407407408, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0196078431372549, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + } + ], + "times": { + "db_healthy": "2026-10-02T13:19:32+00:00", + "db_stopped": "2026-10-02T13:16:30+00:00", + "restart_requested": "2026-10-02T13:19:30+00:00", + "started": "2026-10-02T13:16:14+00:00", + "stop_requested": "2026-10-02T13:16:29+00:00" + }, + "watch_errors": [] +} diff --git a/evidence/slo-quiet-baseline-2026-10-02.jsonl b/evidence/slo-quiet-baseline-2026-10-02.jsonl new file mode 100644 index 0000000..e6029c0 --- /dev/null +++ b/evidence/slo-quiet-baseline-2026-10-02.jsonl @@ -0,0 +1,6 @@ +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:28+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:33+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:38+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:43+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:48+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:53+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} diff --git a/evidence/slo-short-db-2026-10-02.json b/evidence/slo-short-db-2026-10-02.json new file mode 100644 index 0000000..8f6090e --- /dev/null +++ b/evidence/slo-short-db-2026-10-02.json @@ -0,0 +1,1584 @@ +{ + "case": "short", + "events": [ + [ + "2026-10-02T13:13:13+00:00", + 401 + ], + [ + "2026-10-02T13:13:13+00:00", + 401 + ], + [ + "2026-10-02T13:13:14+00:00", + 401 + ], + [ + "2026-10-02T13:13:14+00:00", + 401 + ], + [ + "2026-10-02T13:13:15+00:00", + 401 + ], + [ + "2026-10-02T13:13:15+00:00", + 401 + ], + [ + "2026-10-02T13:13:16+00:00", + 401 + ], + [ + "2026-10-02T13:13:16+00:00", + 401 + ], + [ + "2026-10-02T13:13:16+00:00", + 401 + ], + [ + "2026-10-02T13:13:17+00:00", + 401 + ], + [ + "2026-10-02T13:13:17+00:00", + 401 + ], + [ + "2026-10-02T13:13:18+00:00", + 401 + ], + [ + "2026-10-02T13:13:18+00:00", + 401 + ], + [ + "2026-10-02T13:13:18+00:00", + 401 + ], + [ + "2026-10-02T13:13:19+00:00", + 401 + ], + [ + "2026-10-02T13:13:19+00:00", + 401 + ], + [ + "2026-10-02T13:13:20+00:00", + 401 + ], + [ + "2026-10-02T13:13:20+00:00", + 401 + ], + [ + "2026-10-02T13:13:20+00:00", + 401 + ], + [ + "2026-10-02T13:13:21+00:00", + 401 + ], + [ + "2026-10-02T13:13:21+00:00", + 401 + ], + [ + "2026-10-02T13:13:22+00:00", + 401 + ], + [ + "2026-10-02T13:13:22+00:00", + 401 + ], + [ + "2026-10-02T13:13:23+00:00", + 401 + ], + [ + "2026-10-02T13:13:23+00:00", + 401 + ], + [ + "2026-10-02T13:13:23+00:00", + 401 + ], + [ + "2026-10-02T13:13:24+00:00", + 401 + ], + [ + "2026-10-02T13:13:24+00:00", + 401 + ], + [ + "2026-10-02T13:13:25+00:00", + 401 + ], + [ + "2026-10-02T13:13:25+00:00", + 401 + ], + [ + "2026-10-02T13:13:25+00:00", + 401 + ], + [ + "2026-10-02T13:13:26+00:00", + 401 + ], + [ + "2026-10-02T13:13:26+00:00", + 401 + ], + [ + "2026-10-02T13:13:27+00:00", + 401 + ], + [ + "2026-10-02T13:13:27+00:00", + 401 + ], + [ + "2026-10-02T13:13:27+00:00", + 401 + ], + [ + "2026-10-02T13:13:28+00:00", + 401 + ], + [ + "2026-10-02T13:13:33+00:00", + 503 + ], + [ + "2026-10-02T13:13:39+00:00", + 503 + ], + [ + "2026-10-02T13:13:44+00:00", + 503 + ], + [ + "2026-10-02T13:13:49+00:00", + 503 + ], + [ + "2026-10-02T13:13:55+00:00", + 503 + ], + [ + "2026-10-02T13:14:00+00:00", + 503 + ], + [ + "2026-10-02T13:14:06+00:00", + 503 + ], + [ + "2026-10-02T13:14:11+00:00", + 503 + ], + [ + "2026-10-02T13:14:16+00:00", + 503 + ], + [ + "2026-10-02T13:14:22+00:00", + 503 + ], + [ + "2026-10-02T13:14:27+00:00", + 503 + ], + [ + "2026-10-02T13:14:33+00:00", + 503 + ], + [ + "2026-10-02T13:14:35+00:00", + 401 + ], + [ + "2026-10-02T13:14:35+00:00", + 401 + ], + [ + "2026-10-02T13:14:35+00:00", + 401 + ], + [ + "2026-10-02T13:14:36+00:00", + 401 + ], + [ + "2026-10-02T13:14:36+00:00", + 401 + ], + [ + "2026-10-02T13:14:37+00:00", + 401 + ], + [ + "2026-10-02T13:14:37+00:00", + 401 + ], + [ + "2026-10-02T13:14:37+00:00", + 401 + ], + [ + "2026-10-02T13:14:38+00:00", + 401 + ], + [ + "2026-10-02T13:14:38+00:00", + 401 + ], + [ + "2026-10-02T13:14:39+00:00", + 401 + ], + [ + "2026-10-02T13:14:39+00:00", + 401 + ], + [ + "2026-10-02T13:14:39+00:00", + 401 + ], + [ + "2026-10-02T13:14:40+00:00", + 401 + ], + [ + "2026-10-02T13:14:40+00:00", + 401 + ], + [ + "2026-10-02T13:14:41+00:00", + 401 + ], + [ + "2026-10-02T13:14:41+00:00", + 401 + ], + [ + "2026-10-02T13:14:41+00:00", + 401 + ], + [ + "2026-10-02T13:14:42+00:00", + 401 + ], + [ + "2026-10-02T13:14:42+00:00", + 401 + ], + [ + "2026-10-02T13:14:43+00:00", + 401 + ], + [ + "2026-10-02T13:14:43+00:00", + 401 + ], + [ + "2026-10-02T13:14:43+00:00", + 401 + ], + [ + "2026-10-02T13:14:44+00:00", + 401 + ], + [ + "2026-10-02T13:14:44+00:00", + 401 + ], + [ + "2026-10-02T13:14:45+00:00", + 401 + ], + [ + "2026-10-02T13:14:45+00:00", + 401 + ], + [ + "2026-10-02T13:14:45+00:00", + 401 + ], + [ + "2026-10-02T13:14:46+00:00", + 401 + ], + [ + "2026-10-02T13:14:46+00:00", + 401 + ], + [ + "2026-10-02T13:14:47+00:00", + 401 + ], + [ + "2026-10-02T13:14:47+00:00", + 401 + ], + [ + "2026-10-02T13:14:48+00:00", + 401 + ], + [ + "2026-10-02T13:14:48+00:00", + 401 + ], + [ + "2026-10-02T13:14:48+00:00", + 401 + ], + [ + "2026-10-02T13:14:49+00:00", + 401 + ], + [ + "2026-10-02T13:14:49+00:00", + 401 + ], + [ + "2026-10-02T13:14:50+00:00", + 401 + ], + [ + "2026-10-02T13:14:50+00:00", + 401 + ], + [ + "2026-10-02T13:14:50+00:00", + 401 + ], + [ + "2026-10-02T13:14:51+00:00", + 401 + ], + [ + "2026-10-02T13:14:51+00:00", + 401 + ], + [ + "2026-10-02T13:14:52+00:00", + 401 + ], + [ + "2026-10-02T13:14:52+00:00", + 401 + ], + [ + "2026-10-02T13:14:52+00:00", + 401 + ], + [ + "2026-10-02T13:14:53+00:00", + 401 + ], + [ + "2026-10-02T13:14:53+00:00", + 401 + ], + [ + "2026-10-02T13:14:54+00:00", + 401 + ], + [ + "2026-10-02T13:14:54+00:00", + 401 + ], + [ + "2026-10-02T13:14:54+00:00", + 401 + ], + [ + "2026-10-02T13:14:55+00:00", + 401 + ], + [ + "2026-10-02T13:14:55+00:00", + 401 + ], + [ + "2026-10-02T13:14:56+00:00", + 401 + ], + [ + "2026-10-02T13:14:56+00:00", + 401 + ], + [ + "2026-10-02T13:14:56+00:00", + 401 + ], + [ + "2026-10-02T13:14:57+00:00", + 401 + ], + [ + "2026-10-02T13:14:57+00:00", + 401 + ], + [ + "2026-10-02T13:14:58+00:00", + 401 + ], + [ + "2026-10-02T13:14:58+00:00", + 401 + ], + [ + "2026-10-02T13:14:59+00:00", + 401 + ], + [ + "2026-10-02T13:14:59+00:00", + 401 + ], + [ + "2026-10-02T13:14:59+00:00", + 401 + ], + [ + "2026-10-02T13:15:00+00:00", + 401 + ], + [ + "2026-10-02T13:15:00+00:00", + 401 + ], + [ + "2026-10-02T13:15:01+00:00", + 401 + ], + [ + "2026-10-02T13:15:01+00:00", + 401 + ], + [ + "2026-10-02T13:15:01+00:00", + 401 + ], + [ + "2026-10-02T13:15:02+00:00", + 401 + ], + [ + "2026-10-02T13:15:02+00:00", + 401 + ], + [ + "2026-10-02T13:15:03+00:00", + 401 + ], + [ + "2026-10-02T13:15:03+00:00", + 401 + ], + [ + "2026-10-02T13:15:03+00:00", + 401 + ], + [ + "2026-10-02T13:15:04+00:00", + 401 + ], + [ + "2026-10-02T13:15:04+00:00", + 401 + ], + [ + "2026-10-02T13:15:05+00:00", + 401 + ], + [ + "2026-10-02T13:15:05+00:00", + 401 + ], + [ + "2026-10-02T13:15:05+00:00", + 401 + ], + [ + "2026-10-02T13:15:06+00:00", + 401 + ], + [ + "2026-10-02T13:15:06+00:00", + 401 + ], + [ + "2026-10-02T13:15:07+00:00", + 401 + ], + [ + "2026-10-02T13:15:07+00:00", + 401 + ], + [ + "2026-10-02T13:15:07+00:00", + 401 + ], + [ + "2026-10-02T13:15:08+00:00", + 401 + ], + [ + "2026-10-02T13:15:08+00:00", + 401 + ], + [ + "2026-10-02T13:15:09+00:00", + 401 + ], + [ + "2026-10-02T13:15:09+00:00", + 401 + ], + [ + "2026-10-02T13:15:09+00:00", + 401 + ], + [ + "2026-10-02T13:15:10+00:00", + 401 + ], + [ + "2026-10-02T13:15:10+00:00", + 401 + ], + [ + "2026-10-02T13:15:11+00:00", + 401 + ], + [ + "2026-10-02T13:15:11+00:00", + 401 + ], + [ + "2026-10-02T13:15:11+00:00", + 401 + ], + [ + "2026-10-02T13:15:12+00:00", + 401 + ], + [ + "2026-10-02T13:15:12+00:00", + 401 + ], + [ + "2026-10-02T13:15:13+00:00", + 401 + ], + [ + "2026-10-02T13:15:13+00:00", + 401 + ], + [ + "2026-10-02T13:15:14+00:00", + 401 + ], + [ + "2026-10-02T13:15:14+00:00", + 401 + ], + [ + "2026-10-02T13:15:14+00:00", + 401 + ], + [ + "2026-10-02T13:15:15+00:00", + 401 + ], + [ + "2026-10-02T13:15:15+00:00", + 401 + ], + [ + "2026-10-02T13:15:16+00:00", + 401 + ], + [ + "2026-10-02T13:15:16+00:00", + 401 + ], + [ + "2026-10-02T13:15:16+00:00", + 401 + ], + [ + "2026-10-02T13:15:17+00:00", + 401 + ], + [ + "2026-10-02T13:15:17+00:00", + 401 + ], + [ + "2026-10-02T13:15:18+00:00", + 401 + ], + [ + "2026-10-02T13:15:18+00:00", + 401 + ], + [ + "2026-10-02T13:15:18+00:00", + 401 + ], + [ + "2026-10-02T13:15:19+00:00", + 401 + ], + [ + "2026-10-02T13:15:19+00:00", + 401 + ], + [ + "2026-10-02T13:15:20+00:00", + 401 + ], + [ + "2026-10-02T13:15:20+00:00", + 401 + ], + [ + "2026-10-02T13:15:20+00:00", + 401 + ], + [ + "2026-10-02T13:15:21+00:00", + 401 + ], + [ + "2026-10-02T13:15:21+00:00", + 401 + ], + [ + "2026-10-02T13:15:22+00:00", + 401 + ], + [ + "2026-10-02T13:15:22+00:00", + 401 + ], + [ + "2026-10-02T13:15:22+00:00", + 401 + ], + [ + "2026-10-02T13:15:23+00:00", + 401 + ], + [ + "2026-10-02T13:15:23+00:00", + 401 + ], + [ + "2026-10-02T13:15:24+00:00", + 401 + ], + [ + "2026-10-02T13:15:24+00:00", + 401 + ], + [ + "2026-10-02T13:15:25+00:00", + 401 + ], + [ + "2026-10-02T13:15:25+00:00", + 401 + ], + [ + "2026-10-02T13:15:25+00:00", + 401 + ], + [ + "2026-10-02T13:15:26+00:00", + 401 + ], + [ + "2026-10-02T13:15:26+00:00", + 401 + ], + [ + "2026-10-02T13:15:27+00:00", + 401 + ], + [ + "2026-10-02T13:15:27+00:00", + 401 + ], + [ + "2026-10-02T13:15:27+00:00", + 401 + ], + [ + "2026-10-02T13:15:28+00:00", + 401 + ], + [ + "2026-10-02T13:15:28+00:00", + 401 + ], + [ + "2026-10-02T13:15:29+00:00", + 401 + ], + [ + "2026-10-02T13:15:29+00:00", + 401 + ], + [ + "2026-10-02T13:15:29+00:00", + 401 + ], + [ + "2026-10-02T13:15:30+00:00", + 401 + ], + [ + "2026-10-02T13:15:30+00:00", + 401 + ], + [ + "2026-10-02T13:15:31+00:00", + 401 + ], + [ + "2026-10-02T13:15:31+00:00", + 401 + ], + [ + "2026-10-02T13:15:31+00:00", + 401 + ], + [ + "2026-10-02T13:15:32+00:00", + 401 + ], + [ + "2026-10-02T13:15:32+00:00", + 401 + ], + [ + "2026-10-02T13:15:33+00:00", + 401 + ], + [ + "2026-10-02T13:15:33+00:00", + 401 + ], + [ + "2026-10-02T13:15:33+00:00", + 401 + ], + [ + "2026-10-02T13:15:34+00:00", + 401 + ], + [ + "2026-10-02T13:15:34+00:00", + 401 + ], + [ + "2026-10-02T13:15:35+00:00", + 401 + ], + [ + "2026-10-02T13:15:35+00:00", + 401 + ], + [ + "2026-10-02T13:15:35+00:00", + 401 + ], + [ + "2026-10-02T13:15:36+00:00", + 401 + ], + [ + "2026-10-02T13:15:36+00:00", + 401 + ], + [ + "2026-10-02T13:15:37+00:00", + 401 + ], + [ + "2026-10-02T13:15:37+00:00", + 401 + ], + [ + "2026-10-02T13:15:38+00:00", + 401 + ], + [ + "2026-10-02T13:15:38+00:00", + 401 + ], + [ + "2026-10-02T13:15:38+00:00", + 401 + ], + [ + "2026-10-02T13:15:39+00:00", + 401 + ], + [ + "2026-10-02T13:15:39+00:00", + 401 + ], + [ + "2026-10-02T13:15:40+00:00", + 401 + ], + [ + "2026-10-02T13:15:40+00:00", + 401 + ], + [ + "2026-10-02T13:15:40+00:00", + 401 + ], + [ + "2026-10-02T13:15:41+00:00", + 401 + ], + [ + "2026-10-02T13:15:41+00:00", + 401 + ], + [ + "2026-10-02T13:15:42+00:00", + 401 + ], + [ + "2026-10-02T13:15:42+00:00", + 401 + ], + [ + "2026-10-02T13:15:42+00:00", + 401 + ], + [ + "2026-10-02T13:15:43+00:00", + 401 + ], + [ + "2026-10-02T13:15:43+00:00", + 401 + ], + [ + "2026-10-02T13:15:44+00:00", + 401 + ], + [ + "2026-10-02T13:15:44+00:00", + 401 + ], + [ + "2026-10-02T13:15:44+00:00", + 401 + ], + [ + "2026-10-02T13:15:45+00:00", + 401 + ], + [ + "2026-10-02T13:15:45+00:00", + 401 + ], + [ + "2026-10-02T13:15:46+00:00", + 401 + ], + [ + "2026-10-02T13:15:46+00:00", + 401 + ], + [ + "2026-10-02T13:15:46+00:00", + 401 + ], + [ + "2026-10-02T13:15:47+00:00", + 401 + ], + [ + "2026-10-02T13:15:47+00:00", + 401 + ], + [ + "2026-10-02T13:15:48+00:00", + 401 + ], + [ + "2026-10-02T13:15:48+00:00", + 401 + ], + [ + "2026-10-02T13:15:48+00:00", + 401 + ], + [ + "2026-10-02T13:15:49+00:00", + 401 + ], + [ + "2026-10-02T13:15:49+00:00", + 401 + ], + [ + "2026-10-02T13:15:50+00:00", + 401 + ], + [ + "2026-10-02T13:15:50+00:00", + 401 + ], + [ + "2026-10-02T13:15:51+00:00", + 401 + ], + [ + "2026-10-02T13:15:51+00:00", + 401 + ], + [ + "2026-10-02T13:15:51+00:00", + 401 + ], + [ + "2026-10-02T13:15:52+00:00", + 401 + ], + [ + "2026-10-02T13:15:52+00:00", + 401 + ], + [ + "2026-10-02T13:15:53+00:00", + 401 + ], + [ + "2026-10-02T13:15:53+00:00", + 401 + ], + [ + "2026-10-02T13:15:53+00:00", + 401 + ], + [ + "2026-10-02T13:15:54+00:00", + 401 + ], + [ + "2026-10-02T13:15:54+00:00", + 401 + ], + [ + "2026-10-02T13:15:55+00:00", + 401 + ], + [ + "2026-10-02T13:15:55+00:00", + 401 + ], + [ + "2026-10-02T13:15:55+00:00", + 401 + ], + [ + "2026-10-02T13:15:56+00:00", + 401 + ], + [ + "2026-10-02T13:15:56+00:00", + 401 + ], + [ + "2026-10-02T13:15:57+00:00", + 401 + ], + [ + "2026-10-02T13:15:57+00:00", + 401 + ], + [ + "2026-10-02T13:15:57+00:00", + 401 + ], + [ + "2026-10-02T13:15:58+00:00", + 401 + ] + ], + "planned_seconds": { + "outage": 50, + "post": 90, + "pre": 15 + }, + "prometheus_samples": [ + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:13+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:18+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:23+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:28+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:33+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:38+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:43+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.05239864559019707, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:48+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.10986596449397801, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:53+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.1914814703057944, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:58+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:03+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:08+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:13+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:18+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:23+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:28+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:33+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:38+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:43+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.25, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:48+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.1111111111111111, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:53+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.05263157894736842, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:58+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.02, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:03+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:08+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:13+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:18+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:23+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:28+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:33+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:38+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:43+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:48+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + } + ], + "times": { + "db_healthy": "2026-10-02T13:14:21+00:00", + "db_stopped": "2026-10-02T13:13:28+00:00", + "restart_requested": "2026-10-02T13:14:18+00:00", + "started": "2026-10-02T13:13:13+00:00", + "stop_requested": "2026-10-02T13:13:28+00:00" + }, + "watch_errors": [] +} diff --git a/evidence/slo-thresholds-2026-10-02.md b/evidence/slo-thresholds-2026-10-02.md new file mode 100644 index 0000000..1ecb8be --- /dev/null +++ b/evidence/slo-thresholds-2026-10-02.md @@ -0,0 +1,49 @@ +# Зафиксированные SLO до инъекций отказов — 2026-10-02 + +Стенд: Colima/arm64, 2 CPU, 2 GiB RAM, две API-реплики за HAProxy, +serial worker, PostgreSQL/MinIO/RabbitMQ, Collector/Prometheus/Loki/Tempo/ +Grafana. На момент фиксации fault matrix из перенесённой lab5 выполнена, но +**SLO-инциденты ЛР6 ещё не запускались**. Правила ниже лежат в +`infra/prometheus/rules.yml`; `promtool check rules` обнаружил 6 правил. + +## Baseline + +- 30,010 с, четыре closed-loop клиента, 18 260 успешных POST сообщений, + 0 ошибок, 608,462 RPS, latency успехов p50/p95/p99 + 4,289/14,966/59,664 мс: `lab6-http-baseline-2api-2026-10-02.json`. +- 20 PNG 320×200 (111 817 байт) + 20 PNG 800×600 (1 441 103 байта), + четыре клиента: 40/40 `ready`, 0 ошибок, 13,029 с. p95 от начала upload + до ready 1,305/1,618 с по профилям: `lab6-image-baseline-2api-2026-10-02.json`. + Poll раз в 0,3 с; SDK histogram измеряет от durable job creation до result + COMMIT и не включает upload до принятия. + +## Цели и формулы + +- **API:** 99% бизнес HTTP-запросов без 5xx за скользящие 10 минут. + Numerator — 5xx, denominator — все non-health ответы, включая 4xx; + HAProxy-generated 503 не видны SDK (ограничение), поэтому эксперимент + обращается к живому API-процессу напрямую при DB outage. При нулевом + traffic SLI неизвестен, а не равен 100%. +- **Ready:** 95% завершённых image jobs достигают `ready` не позднее 5 секунд + после durable acceptance за скользящие 10 минут. Числитель плохих jobs: + `rate(ready_duration_count) - rate(ready_duration_bucket{le="5"})`, + знаменатель — `rate(ready_duration_count)`. Застрявшие jobs ещё не + завершены и не попадают в histogram; queue oldest age и отдельный alert + закрывают слепую зону. + +Локальные burn-rate окна уменьшены пропорционально времени демонстрации: +short требует одновременно 15 и 30 секунд, threshold 14,4 бюджета и `for:10s`; +long требует одновременно 60 и 120 секунд, threshold 6 и `for:90s`. +Для API error budget 1% это error ratio >14,4% / >6%; для ready budget 5% +slow ratio >72% / >30%. Оценка правил каждые 5 секунд. Operational alerts: +HTTP 5xx >10% за 30 с при `for:10s`, oldest queued age >10 с при +`for:10s` и sample freshness <15 с. Порог выбран до сценариев; +эксперимент не меняет его после наблюдений. + +Профиль опыта: одинаковые API/worker/DB лимиты и синтетические данные. +Короткий сбой ожидается увидеть short-правилом без long; длительное `for` +подавляет long на кратких всплесках. Длинный сбой должен вызвать оба. +Записываются UTC timestamps воздействия, pending/firing/inactive из +Prometheus rule API каждые 5 с, detection/recovery, false positives на +спокойном baseline и выбранные SLI/queue values. Эти окна учебные и не +являются production SLO. diff --git a/evidence/telemetry-backend-outage-2026-10-02.json b/evidence/telemetry-backend-outage-2026-10-02.json new file mode 100644 index 0000000..3ead1ad --- /dev/null +++ b/evidence/telemetry-backend-outage-2026-10-02.json @@ -0,0 +1,150 @@ +{ + "collector_counter_delta": { + "otelcol_exporter_enqueue_failed_spans": 19486.0, + "otelcol_exporter_send_failed_log_records": 11063.0, + "otelcol_exporter_send_failed_metric_points": 484.0, + "otelcol_exporter_send_failed_spans": 16547.0, + "otelcol_receiver_refused_log_records": 0.0, + "otelcol_receiver_refused_metric_points": 0.0, + "otelcol_receiver_refused_spans": 0.0 + }, + "collector_counters_after": { + "otelcol_exporter_enqueue_failed_spans": 112097.0, + "otelcol_exporter_send_failed_log_records": 31979.0, + "otelcol_exporter_send_failed_metric_points": 1570.0, + "otelcol_exporter_send_failed_spans": 44902.0, + "otelcol_receiver_refused_log_records": 0.0, + "otelcol_receiver_refused_metric_points": 0.0, + "otelcol_receiver_refused_spans": 0.0 + }, + "collector_counters_before": { + "otelcol_exporter_enqueue_failed_spans": 92611.0, + "otelcol_exporter_send_failed_log_records": 20916.0, + "otelcol_exporter_send_failed_metric_points": 1086.0, + "otelcol_exporter_send_failed_spans": 28355.0, + "otelcol_receiver_refused_log_records": 0.0, + "otelcol_receiver_refused_metric_points": 0.0, + "otelcol_receiver_refused_spans": 0.0 + }, + "load_passed": true, + "smoke_passed": true, + "smoke_tail": [ + "python3 tests/smoke.py --base-url \"http://127.0.0.1:8080\" ", + "PASS: lab6 public API scenario; infrastructure and grading evidence remain separate." + ], + "started": "2026-10-02T14:01:35+00:00", + "stats_after_35s": [ + { + "BlockIO": "2.23GB / 0B", + "CPUPerc": "1.21%", + "Container": "mtusi2026lab6-otel-collector-1", + "ID": "3bedc51054f5", + "MemPerc": "30.20%", + "MemUsage": "116MiB / 384MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "126MB / 37.1MB", + "PIDs": "10" + }, + { + "BlockIO": "40.6MB / 0B", + "CPUPerc": "14.25%", + "Container": "mtusi2026lab6-api-1", + "ID": "2ad72f3266ec", + "MemPerc": "32.39%", + "MemUsage": "82.91MiB / 256MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "22.6MB / 29.5MB", + "PIDs": "10" + }, + { + "BlockIO": "11.3MB / 0B", + "CPUPerc": "0.09%", + "Container": "mtusi2026lab6-worker-1", + "ID": "88e67b14961c", + "MemPerc": "7.12%", + "MemUsage": "54.67MiB / 768MiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "334kB / 453kB", + "PIDs": "8" + } + ], + "stats_before": [ + { + "BlockIO": "2.21GB / 0B", + "CPUPerc": "0.19%", + "Container": "mtusi2026lab6-otel-collector-1", + "ID": "3bedc51054f5", + "MemPerc": "18.44%", + "MemUsage": "70.82MiB / 384MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "110MB / 36.8MB", + "PIDs": "10" + }, + { + "BlockIO": "35.6MB / 0B", + "CPUPerc": "8.60%", + "Container": "mtusi2026lab6-api-1", + "ID": "2ad72f3266ec", + "MemPerc": "22.86%", + "MemUsage": "58.52MiB / 256MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "1.13MB / 1.78MB", + "PIDs": "10" + }, + { + "BlockIO": "10.8MB / 0B", + "CPUPerc": "0.15%", + "Container": "mtusi2026lab6-worker-1", + "ID": "88e67b14961c", + "MemPerc": "7.09%", + "MemUsage": "54.44MiB / 768MiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "188kB / 267kB", + "PIDs": "8" + } + ], + "stats_during": [ + { + "BlockIO": "2.23GB / 0B", + "CPUPerc": "2.40%", + "Container": "mtusi2026lab6-otel-collector-1", + "ID": "3bedc51054f5", + "MemPerc": "27.14%", + "MemUsage": "104.2MiB / 384MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "126MB / 37MB", + "PIDs": "10" + }, + { + "BlockIO": "40.6MB / 0B", + "CPUPerc": "7.68%", + "Container": "mtusi2026lab6-api-1", + "ID": "2ad72f3266ec", + "MemPerc": "32.65%", + "MemUsage": "83.58MiB / 256MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "22.5MB / 29.3MB", + "PIDs": "13" + }, + { + "BlockIO": "11.3MB / 0B", + "CPUPerc": "0.14%", + "Container": "mtusi2026lab6-worker-1", + "ID": "88e67b14961c", + "MemPerc": "7.12%", + "MemUsage": "54.67MiB / 768MiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "325kB / 431kB", + "PIDs": "8" + } + ], + "stores": [ + "prometheus", + "loki", + "tempo" + ], + "stores_healthy": true, + "stores_restored": true, + "stores_restored_at": "2026-10-02T14:03:01+00:00", + "stores_stopped": "2026-10-02T14:01:48+00:00" +} diff --git a/evidence/telemetry-outage-http-load-2026-10-02.json b/evidence/telemetry-outage-http-load-2026-10-02.json new file mode 100644 index 0000000..a81e211 --- /dev/null +++ b/evidence/telemetry-outage-http-load-2026-10-02.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.017, + "concurrency": 4, + "successful_requests": 17876, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 595.531, + "successful_latency_ms": { + "p50": 4.844, + "p95": 16.108, + "p99": 49.174 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/tempo-recovery-load-2026-10-02.json b/evidence/tempo-recovery-load-2026-10-02.json new file mode 100644 index 0000000..ec0c08d --- /dev/null +++ b/evidence/tempo-recovery-load-2026-10-02.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.011, + "concurrency": 4, + "successful_requests": 18860, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 628.445, + "successful_latency_ms": { + "p50": 4.677, + "p95": 14.469, + "p99": 44.625 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/infra/bootstrap-s3.sh b/infra/bootstrap-s3.sh new file mode 100644 index 0000000..becaf21 --- /dev/null +++ b/infra/bootstrap-s3.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu + +read_secret() { + tr -d '\n' < "/run/secrets/$1" +} + +mc alias set local http://s3:9000 "$(read_secret minio_root_user)" "$(read_secret minio_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +mc admin policy create local messenger-api /policies/minio-api-policy.json >/dev/null +mc admin policy create local messenger-maintenance /policies/minio-maintenance-policy.json >/dev/null +mc admin user add local "$(read_secret s3_access_key_id)" "$(read_secret s3_secret_access_key)" >/dev/null +mc admin policy attach local messenger-api --user "$(read_secret s3_access_key_id)" >/dev/null +mc admin user add local "$(read_secret s3_maintenance_key_id)" "$(read_secret s3_maintenance_secret_key)" >/dev/null +mc admin policy attach local messenger-maintenance --user "$(read_secret s3_maintenance_key_id)" >/dev/null diff --git a/infra/grafana/dashboards/telemetry.json b/infra/grafana/dashboards/telemetry.json new file mode 100644 index 0000000..d4d3559 --- /dev/null +++ b/infra/grafana/dashboards/telemetry.json @@ -0,0 +1,606 @@ +{ + "uid": "lab6-telemetry", + "title": "Lab 6 · Messenger signals", + "tags": [ + "lab6" + ], + "timezone": "browser", + "schemaVersion": 40, + "version": 2, + "refresh": "10s", + "time": { + "from": "now-1h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Stores", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "up{job=~\"prometheus|otel-collector|loki|tempo\"}", + "legendFormat": "{{job}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "type": "timeseries", + "title": "HTTP RPS", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + } + }, + { + "id": 3, + "type": "timeseries", + "title": "HTTP 5xx ratio", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001)", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 4, + "type": "timeseries", + "title": "HTTP p50/p95/p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.50,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "series 2" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "series 3" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Job outcomes", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(outcome)(rate(messenger_jobs_processed_total[1m]))", + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Job p95/p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_jobs_duration_seconds_bucket[5m])))", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_jobs_duration_seconds_bucket[5m])))", + "legendFormat": "series 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Queue depth", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_queue_depth{queue_name=\"attachments.process\"} and (time()-timestamp(messenger_queue_depth{queue_name=\"attachments.process\"})<15))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Oldest queued job", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_oldest_age_seconds{queue_name=\"attachments.process\"} and (time()-timestamp(messenger_queue_oldest_age_seconds{queue_name=\"attachments.process\"})<15))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + } + }, + { + "id": 9, + "type": "timeseries", + "title": "DB pool used / waiting", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name)(messenger_db_pool_used)", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "sum by(service_name)(messenger_db_pool_waiting)", + "legendFormat": "series 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Worker busy / capacity", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_worker_busy)", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "sum(messenger_worker_capacity)", + "legendFormat": "series 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Time to ready p50/p95/p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.50,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))", + "legendFormat": "series 2" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))", + "legendFormat": "series 3" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Process CPU", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name)(rate(messenger_process_cpu_time_seconds_total[1m]))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Process RSS", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name)(messenger_process_rss_bytes)", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Collector queue / failures", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "otelcol_exporter_queue_size{job=\"otel-collector\"}", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "sum(rate(otelcol_exporter_send_failed_metric_points[1m]))", + "legendFormat": "series 2" + }, + { + "refId": "C", + "expr": "sum(rate(otelcol_exporter_send_failed_log_records[1m]))", + "legendFormat": "series 3" + }, + { + "refId": "D", + "expr": "sum(rate(otelcol_exporter_send_failed_spans[1m]))", + "legendFormat": "series 4" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Alerts firing", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertstate=\"firing\",alertname=~\"Lab6.*\"}", + "legendFormat": "{{alertname}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + } + }, + { + "id": 16, + "type": "timeseries", + "title": "HTTP 4xx RPS (client responses)", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [{ + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{status_class=\"4xx\",route!~\"/health/.*\"}[1m])) or vector(0)", + "legendFormat": "4xx" + }], + "fieldConfig": {"defaults": {"unit": "reqps", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 56} + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..e172240 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: + - name: Lab 6 + type: file + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..f03287e --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,18 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..40952ab --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,26 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + timeout connect 2s + timeout client 15s + timeout server 15s + +resolvers docker + nameserver docker 127.0.0.11:53 + resolve_retries 2 + timeout retry 1s + hold valid 1s + +frontend public + bind *:8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + server-template api 1-4 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr none diff --git a/infra/loki/config.yaml b/infra/loki/config.yaml new file mode 100644 index 0000000..508633a --- /dev/null +++ b/infra/loki/config.yaml @@ -0,0 +1,32 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + +analytics: + reporting_enabled: false diff --git a/infra/minio-api-policy.json b/infra/minio-api-policy.json new file mode 100644 index 0000000..54be94d --- /dev/null +++ b/infra/minio-api-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + } + ] +} diff --git a/infra/minio-maintenance-policy.json b/infra/minio-maintenance-policy.json new file mode 100644 index 0000000..6beb6bd --- /dev/null +++ b/infra/minio-maintenance-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + } + ] +} diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..013d283 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,72 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + timeout: 2s + send_batch_size: 512 + +exporters: + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + remote_write_queue: + enabled: true + queue_size: 1000 + num_consumers: 1 + retry_on_failure: + enabled: true + max_elapsed_time: 30s + otlphttp/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + queue_size: 64 + retry_on_failure: + enabled: true + max_elapsed_time: 30s + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + sending_queue: + enabled: true + queue_size: 64 + retry_on_failure: + enabled: true + max_elapsed_time: 30s + +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + without_type_suffix: true + without_units: true + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..01012e9 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,20 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +rule_files: + - /etc/prometheus/rules.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + - job_name: loki + static_configs: + - targets: ["loki:3100"] + - job_name: tempo + static_configs: + - targets: ["tempo:3200"] diff --git a/infra/prometheus/rules.yml b/infra/prometheus/rules.yml new file mode 100644 index 0000000..1b0729d --- /dev/null +++ b/infra/prometheus/rules.yml @@ -0,0 +1,91 @@ +groups: + - name: lab6-operational + interval: 5s + rules: + - alert: Lab6HighHttpErrorRate + expr: | + sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])), 0.001) > 0.10 + for: 10s + labels: + severity: warning + annotations: + summary: More than 10% of business HTTP requests return 5xx + runbook: See infra/runbooks/incident.md#http-errors + - alert: Lab6QueueLag + expr: | + (messenger_queue_oldest_age_seconds{queue_name="attachments.process"} + and (time() - timestamp(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}) < 15)) > 10 + for: 10s + labels: + severity: warning + annotations: + summary: Oldest queued image job is over 10 seconds old + runbook: See infra/runbooks/incident.md#queue-lag + - name: lab6-slo-burn + interval: 5s + rules: + - alert: Lab6ApiBurnShort + expr: | + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[15s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[15s])),0.001) > 0.144) + and + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001) > 0.144) + for: 10s + labels: + severity: page + slo: api + window: short + annotations: + summary: API error budget burning over 14.4x in both 15s and 30s windows + runbook: See infra/runbooks/incident.md#http-errors + - alert: Lab6ApiBurnLong + expr: | + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[60s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[60s])),0.001) > 0.06) + and + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[120s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[120s])),0.001) > 0.06) + for: 90s + labels: + severity: warning + slo: api + window: long + annotations: + summary: API error budget burning over 6x in both 60s and 120s windows + runbook: See infra/runbooks/incident.md#http-errors + - alert: Lab6ReadyBurnShort + expr: | + ((sum(rate(messenger_attachments_ready_duration_seconds_count[15s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[15s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[15s])),0.001) > 0.72) + and + ((sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001) > 0.72) + for: 10s + labels: + severity: page + slo: ready + window: short + annotations: + summary: Ready-time error budget burning over 14.4x in both 15s and 30s windows + runbook: See infra/runbooks/incident.md#queue-lag + - alert: Lab6ReadyBurnLong + expr: | + ((sum(rate(messenger_attachments_ready_duration_seconds_count[60s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[60s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[60s])),0.001) > 0.30) + and + ((sum(rate(messenger_attachments_ready_duration_seconds_count[120s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[120s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[120s])),0.001) > 0.30) + for: 90s + labels: + severity: warning + slo: ready + window: long + annotations: + summary: Ready-time error budget burning over 6x in both 60s and 120s windows + runbook: See infra/runbooks/incident.md#queue-lag diff --git a/infra/runbooks/incident.md b/infra/runbooks/incident.md new file mode 100644 index 0000000..5de27a4 --- /dev/null +++ b/infra/runbooks/incident.md @@ -0,0 +1,54 @@ +# Lab 6: local incident runbook + +Use Grafana at `http://localhost:3000` (dashboard **Lab 6 · Messenger signals**) or +Prometheus rules at `/alerts` inside the Compose network. All times below are +UTC. The dashboard uses stored Prometheus data; Tempo and Loki are separate +datasources. Keep the last hour visible before diagnosing a recovered incident. + +## HTTP errors + +`Lab6HighHttpErrorRate` fires when 5xx / all non-health HTTP requests exceeds +10% over 30 seconds for another 10 seconds. Client 4xx are in the denominator +and have a separate panel; they are not server failures. Check HTTP RPS first: +an empty or tiny denominator makes an error ratio unreliable. Compare 5xx +ratio with DB pool used/waiting and `up` for the telemetry stores. In Tempo, +find an `HTTP` server span in the time range and inspect its `db.*` child span +duration/status. Query Loki `{service_name="messenger-api"}` for the same +trace ID. A growing pool wait and failing DB span identify a database incident; +a long server span with fast DB children points to the application path. +Restart PostgreSQL only after confirming its container state and persistent +volume. Confirm `up`, 5xx ratio, and `/health/ready` recover; record first +`pending`, `firing`, and `inactive` timestamps from the rule API. + +## Queue lag + +`Lab6QueueLag` fires when the oldest durable queued job exceeds 10 seconds for +another 10 seconds. `messenger_queue_depth` counts rows with `status='queued'` +in PostgreSQL, including an outbox row awaiting publication. It is sampled by +the publisher. First compare queue depth, oldest age, `messenger_worker_busy` +and `messenger_worker_capacity`. If busy disappears and capacity is absent, +inspect the worker container; if capacity is present but processing outcome +rates stop, inspect broker connectivity and `job_published` events in Loki. +If jobs fail repeatedly, inspect `messenger_jobs_processed_total` grouped by +outcome and the corresponding consumer trace; do not replay until the cause +is corrected. Restart the worker, then verify queued count/age fall, ready +durations appear, and the alert returns to inactive. + +## Follow an upload + +Take the `trace_id` from the API `attachment_uploaded` log in Loki. Tempo +should contain the API upload span, publisher `attachments.publish` producer +span, worker `attachments.process` consumer span and its S3/render/DB children. +The worker `job_processed` log has the same trace ID. Job identifiers are +synthetic test identifiers; metric labels contain only route templates, +status classes, job type/outcome and queue name. Never copy request bodies, +tokens, signed object URLs or raw image bytes into evidence. + +## Missing telemetry + +If a signal is absent, check Collector `otelcol_exporter_queue_size`, +`otelcol_exporter_send_failed_*` and store `up` before concluding the business +service recovered. The SDK uses asynchronous bounded queues and 2-second +export timeouts; Collector queues are finite. Export loss during a backend +outage is expected and must be reported. A missing sample is not zero errors. +Telemetry stores are not part of API readiness. diff --git a/infra/tempo/config.yaml b/infra/tempo/config.yaml new file mode 100644 index 0000000..14f76b5 --- /dev/null +++ b/infra/tempo/config.yaml @@ -0,0 +1,22 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + max_block_bytes: 5000000 + max_block_duration: 1m + concurrent_flushes: 1 + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..dbfb006 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,36 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX chats_history_idx ON chats (created_at, id); + +CREATE TABLE memberships ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 99), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE INDEX memberships_user_idx ON memberships (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES memberships(chat_id, user_id) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_sessions.sql b/migrations/002_sessions.sql new file mode 100644 index 0000000..5d2f364 --- /dev/null +++ b/migrations/002_sessions.sql @@ -0,0 +1,7 @@ +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL +); + +CREATE INDEX sessions_user_idx ON sessions (user_id); diff --git a/migrations/003_message_replays.sql b/migrations/003_message_replays.sql new file mode 100644 index 0000000..e5a8627 --- /dev/null +++ b/migrations/003_message_replays.sql @@ -0,0 +1,11 @@ +CREATE TABLE message_replays ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + request_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid REFERENCES messages(id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, request_key) +); + +CREATE INDEX message_replays_expiry_idx ON message_replays (created_at); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..576f30b --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,32 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (id, chat_id) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at); + +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 0 AND 4000); +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position), + FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE, + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/005_attachment_owner.sql b/migrations/005_attachment_owner.sql new file mode 100644 index 0000000..6db4290 --- /dev/null +++ b/migrations/005_attachment_owner.sql @@ -0,0 +1,2 @@ +ALTER TABLE attachments ADD COLUMN owner_id uuid NOT NULL REFERENCES users(id); +ALTER TABLE attachments ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(); diff --git a/migrations/006_pipeline.sql b/migrations/006_pipeline.sql new file mode 100644 index 0000000..f332a6f --- /dev/null +++ b/migrations/006_pipeline.sql @@ -0,0 +1,74 @@ +-- Lab5: PostgreSQL is the source of truth; RabbitMQ carries only job IDs. +ALTER TABLE attachments + ADD COLUMN staging_key text UNIQUE, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version > 0), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt > 0), + status text NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), + lease_token uuid, + lease_expires_at timestamptz, + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (attachment_id, pipeline_version), + CHECK ((status = 'processing') = + (lease_token IS NOT NULL AND lease_expires_at IS NOT NULL)) +); + +CREATE INDEX attachment_jobs_due_idx + ON attachment_jobs (next_attempt_at, id) WHERE status = 'queued'; +CREATE INDEX attachment_jobs_expired_idx + ON attachment_jobs (lease_expires_at, id) WHERE status = 'processing'; + +-- One logical publication per managed attempt. A crash after broker confirm +-- but before published_at may republish; consumers must tolerate that duplicate. +CREATE TABLE attachment_outbox ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + attempt integer NOT NULL CHECK (attempt > 0), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + claim_token uuid, + claim_expires_at timestamptz, + published_at timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (job_id, attempt), + CHECK ((claim_token IS NULL) = (claim_expires_at IS NULL)) +); + +CREATE INDEX attachment_outbox_pending_idx + ON attachment_outbox (available_at, id) WHERE published_at IS NULL; + +-- Internal 'original' is stored alongside public thumbnail/crop/optimized +-- so every physical effect has the same versioned identity and checksum. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL, + pipeline_version integer NOT NULL, + name text NOT NULL CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')), + object_key text NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, pipeline_version, name), + FOREIGN KEY (attachment_id, pipeline_version) + REFERENCES attachment_jobs (attachment_id, pipeline_version) ON DELETE CASCADE +); + +-- An API upload reserves its staging key before S3 PUT. Cleanup only removes +-- objects without a committed job and without an unexpired upload marker. +CREATE TABLE attachment_uploads ( + id uuid PRIMARY KEY, + staging_key text NOT NULL UNIQUE, + lease_expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachment_uploads_expired_idx ON attachment_uploads (lease_expires_at); diff --git a/migrations/007_replay_audit.sql b/migrations/007_replay_audit.sql new file mode 100644 index 0000000..cbe4c37 --- /dev/null +++ b/migrations/007_replay_audit.sql @@ -0,0 +1,16 @@ +-- A replay starts a new bounded run without deleting earlier outbox history. +ALTER TABLE attachment_jobs + ADD COLUMN replay_start_attempt integer NOT NULL DEFAULT 1 + CHECK (replay_start_attempt > 0 AND replay_start_attempt <= attempt); + +CREATE TABLE attachment_job_replays ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + previous_attempt integer NOT NULL CHECK (previous_attempt > 0), + new_attempt integer NOT NULL CHECK (new_attempt > previous_attempt), + previous_error_code text NOT NULL, + operator_name text NOT NULL CHECK (length(operator_name) BETWEEN 1 AND 80), + reason text NOT NULL CHECK (length(reason) BETWEEN 1 AND 200), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (job_id, new_attempt) +); diff --git a/migrations/008_trace_context.sql b/migrations/008_trace_context.sql new file mode 100644 index 0000000..f992f9a --- /dev/null +++ b/migrations/008_trace_context.sql @@ -0,0 +1,7 @@ +-- Preserve the upload span context across the durable outbox and every retry. +-- Version 007 is reserved for the lab5 replay/audit migration. +ALTER TABLE attachment_jobs + ADD COLUMN traceparent text + CHECK (traceparent IS NULL OR traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'), + ADD COLUMN tracestate text + CHECK (tracestate IS NULL OR length(tracestate) <= 512); diff --git a/migrations/009_message_text_utf8.sql b/migrations/009_message_text_utf8.sql new file mode 100644 index 0000000..86196ee --- /dev/null +++ b/migrations/009_message_text_utf8.sql @@ -0,0 +1,6 @@ +-- PostgreSQL text cannot contain U+0000; bytea preserves the complete UTF-8 payload. +-- API validation enforces the 4000-character limit (up to 16000 UTF-8 bytes). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); +ALTER TABLE messages ADD CONSTRAINT messages_text_check + CHECK (octet_length(text) <= 16000); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2655fc2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +fastapi==0.118.3 +pydantic==2.12.4 +uvicorn==0.38.0 +psycopg[binary,pool]==3.2.12 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +minio==7.2.20 +Pillow==11.3.0 +urllib3==2.8.0 +pika==1.4.4 +opentelemetry-api==1.45.0 +opentelemetry-sdk==1.45.0 +opentelemetry-semantic-conventions==0.66b0 +opentelemetry-exporter-otlp-proto-http==1.45.0 diff --git a/scripts/bootstrap_broker.py b/scripts/bootstrap_broker.py new file mode 100644 index 0000000..25d2780 --- /dev/null +++ b/scripts/bootstrap_broker.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Idempotently declare the durable processing queue via RabbitMQ management API.""" + +import base64 +import json +import os +import time +from urllib.error import HTTPError, URLError +from urllib.parse import quote, unquote, urlsplit +from urllib.request import Request, urlopen + + +broker_url = open(os.environ.get("BROKER_URL_FILE", "/run/secrets/broker_url"), encoding="utf-8").read().strip() +parsed = urlsplit(broker_url) +if parsed.scheme != "amqp" or not parsed.username or not parsed.password: + raise SystemExit("Invalid broker URL file") +credentials = f"{unquote(parsed.username)}:{unquote(parsed.password)}".encode() +headers = { + "Authorization": "Basic " + base64.b64encode(credentials).decode("ascii"), + "Content-Type": "application/json", +} +url = "http://broker:15672/api/queues/%2F/" + quote("attachments.process", safe="") +body = json.dumps( + {"durable": True, "auto_delete": False, "arguments": {"x-queue-type": "quorum"}} +).encode() + +for attempt in range(30): + try: + with urlopen(Request(url, data=body, headers=headers, method="PUT"), timeout=3): + pass + with urlopen(Request(url, headers=headers), timeout=3) as response: + queue = json.load(response) + if queue.get("type") != "quorum" or not queue.get("durable"): + raise SystemExit("Processing queue is not durable quorum") + print("Durable quorum queue attachments.process is ready") + break + except HTTPError as exc: + if exc.code in (400, 401, 403, 406): + raise SystemExit(f"RabbitMQ queue declaration failed (HTTP {exc.code})") from None + except URLError: + pass + if attempt == 29: + raise SystemExit("RabbitMQ management API did not become ready") + time.sleep(1) diff --git a/scripts/init_local_broker.py b/scripts/init_local_broker.py new file mode 100644 index 0000000..e24bf49 --- /dev/null +++ b/scripts/init_local_broker.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Create local RabbitMQ credentials without writing them to tracked files.""" + +from pathlib import Path +from secrets import token_hex +from urllib.parse import quote + + +root = Path(__file__).resolve().parents[1] +secret_dir = root / "secrets" +secret_dir.mkdir(mode=0o700, exist_ok=True) +secret_dir.chmod(0o700) +config = secret_dir / "rabbitmq.conf" +url_file = secret_dir / "broker_url" +username = "lab5_app" + +if config.exists(): + values = dict( + line.split(" = ", 1) + for line in config.read_text(encoding="utf-8").splitlines() + if " = " in line + ) + if values.get("default_user") != username or not values.get("default_pass"): + raise SystemExit("Existing RabbitMQ config cannot be reused") + password = values["default_pass"] +else: + password = token_hex(24) + config.write_text( + f"default_user = {username}\ndefault_pass = {password}\n", + encoding="utf-8", + ) +config.chmod(0o600) +url_file.write_text( + f"amqp://{username}:{quote(password, safe='')}@broker:5672/%2F\n", + encoding="utf-8", +) +url_file.chmod(0o600) +print("Local broker credentials are ready in ignored secrets/") diff --git a/scripts/measure-image-pipeline.py b/scripts/measure-image-pipeline.py new file mode 100644 index 0000000..4a744fe --- /dev/null +++ b/scripts/measure-image-pipeline.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Small synthetic image workload; reports upload and time-to-ready separately.""" + +import argparse +import hashlib +import json +import math +import os +import struct +import sys +import time +import uuid +import zlib +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def png_noise(width, height): + raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height)) + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + return (b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"")) + + +def percentile(values, percentile_value): + values = sorted(values) + return round(values[max(0, math.ceil(len(values) * percentile_value) - 1)], 3) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--per-size", type=int, default=4) + parser.add_argument("--profile", choices=("small_320x200", "noise_800x600")) + parser.add_argument("--output", required=True) + args = parser.parse_args() + if not 1 <= args.concurrency <= 16 or not 1 <= args.per_size <= 100: + parser.error("invalid workload size") + + setup = Client(args.base_url) + alice, _bob, _eve, chat = setup.bootstrap() + images = { + "small_320x200": (ROOT / "tests/fixtures/sample.png").read_bytes(), + "noise_800x600": png_noise(800, 600), + } + if args.profile: + images = {args.profile: images[args.profile]} + items = [(name, data) for name, data in images.items() for _ in range(args.per_size)] + + def work(item): + name, data = item + client = Client(args.base_url, timeout=20) + start = time.monotonic() + response = client.call( + "POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}.png", + 202, actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + accepted = time.monotonic() + attachment = response["json"] + if attachment["sha256"] != hashlib.sha256(data).hexdigest(): + raise Failure("accepted checksum mismatch") + client.wait_ready(alice, attachment, wait_seconds=180) + ready = time.monotonic() + return {"profile": name, "size_bytes": len(data), + "upload_ms": round((accepted - start) * 1000, 3), + "ready_ms": round((ready - start) * 1000, 3)} + + started = time.monotonic() + completed, failures = [], [] + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + for future in as_completed([executor.submit(work, item) for item in items]): + try: + completed.append(future.result()) + except (Failure, OSError, ValueError) as exc: + failures.append(type(exc).__name__) + elapsed = time.monotonic() - started + result = { + "lab": 6, "workload": "synthetic PNG upload and poll until ready", + "concurrency": args.concurrency, "per_size": args.per_size, + "elapsed_seconds": round(elapsed, 3), "accepted_ready": len(completed), + "failed": len(failures), "failure_types": sorted(set(failures)), + "profiles": {}, + } + for name, data in images.items(): + rows = [item for item in completed if item["profile"] == name] + result["profiles"][name] = { + "size_bytes": len(data), "successful": len(rows), + "upload_ms": {f"p{p}": percentile([r["upload_ms"] for r in rows], p / 100) + for p in (50, 95, 99)} if rows else {}, + "ready_ms": {f"p{p}": percentile([r["ready_ms"] for r in rows], p / 100) + for p in (50, 95, 99)} if rows else {}, + } + Path(args.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2, sort_keys=True)) + if failures or len(completed) != len(items): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 0000000..e346d4f --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,30 @@ +"""Apply numbered SQL migrations once, atomically and under a DB lock.""" + +from pathlib import Path +import sys + +import psycopg + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.db import database_url + + +def main() -> None: + migration_dir = Path(__file__).resolve().parents[1] / "migrations" + with psycopg.connect(database_url(), connect_timeout=5, options="-c statement_timeout=30000") as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(2026, 2)") + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())" + ) + for path in sorted(migration_dir.glob("[0-9][0-9][0-9]_*.sql")): + version = int(path.name.split("_", 1)[0]) + if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone(): + continue + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (version,)) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..d8a434a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env stop api lb publisher worker staging-cleanup broker postgres s3 +docker compose --env-file .env up -d --wait --scale api=2 diff --git a/scripts/setup-local-secrets.py b/scripts/setup-local-secrets.py new file mode 100644 index 0000000..8cb5829 --- /dev/null +++ b/scripts/setup-local-secrets.py @@ -0,0 +1,44 @@ +"""Create local Compose secrets without printing or overwriting existing values.""" + +import os +import secrets +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] / "secrets" + + +def write(name: str, value: str) -> None: + fd = os.open(ROOT / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as out: + out.write(value) + + +def main() -> None: + ROOT.mkdir(mode=0o700, exist_ok=True) + db_names = {"postgres_password", "database_url"} + s3_names = { + "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key", + "s3_maintenance_key_id", "s3_maintenance_secret_key", + } + for names in (db_names, s3_names): + existing = {name for name in names if (ROOT / name).exists()} + if existing and existing != names: + raise SystemExit(f"Incomplete secret group: {', '.join(sorted(names - existing))}") + if not (ROOT / "postgres_password").exists(): + password = secrets.token_urlsafe(32) + write("postgres_password", password) + write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") + if not (ROOT / "minio_root_user").exists(): + write("minio_root_user", "lab6admin") + write("minio_root_password", secrets.token_urlsafe(32)) + write("s3_access_key_id", secrets.token_hex(10)) + write("s3_secret_access_key", secrets.token_urlsafe(32)) + write("s3_maintenance_key_id", secrets.token_hex(10)) + write("s3_maintenance_secret_key", secrets.token_urlsafe(32)) + if not (ROOT / "grafana_admin_password").exists(): + write("grafana_admin_password", secrets.token_urlsafe(32)) + + +if __name__ == "__main__": + main() diff --git a/scripts/slo_watch.py b/scripts/slo_watch.py new file mode 100644 index 0000000..1f4b98a --- /dev/null +++ b/scripts/slo_watch.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Capture local Prometheus alert states and SLI values without credentials.""" + +import argparse +import json +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlencode +from urllib.request import urlopen + + +QUERIES = { + "http_5xx_ratio_30s": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001)', + "queue_oldest_seconds": 'max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"})', + "ready_slow_ratio_30s": '(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001)', + "collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans)', +} + + +def fetch(base, path, params=None): + url = base + path + ("?" + urlencode(params) if params else "") + with urlopen(url, timeout=4) as response: + result = json.load(response) + if result["status"] != "success": + raise RuntimeError(result) + return result["data"] + + +def snapshot(base): + groups = fetch(base, "/api/v1/rules", {"type": "alert"})["groups"] + alerts = [rule for group in groups for rule in group["rules"]] + row = { + "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "alerts": {item["name"]: item["state"] + for item in alerts if item["name"].startswith("Lab6")}, + "values": {}, + } + for name, query in QUERIES.items(): + values = fetch(base, "/api/v1/query", {"query": query})["result"] + row["values"][name] = float(values[0]["value"][1]) if values else None + return row + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prometheus", default="http://127.0.0.1:9090") + parser.add_argument("--duration", type=int, default=60) + parser.add_argument("--interval", type=int, default=5) + parser.add_argument("--output", required=True) + args = parser.parse_args() + if not 5 <= args.duration <= 900 or not 1 <= args.interval <= 30: + parser.error("invalid duration or interval") + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + end = time.monotonic() + args.duration + with path.open("w", encoding="utf-8") as out: + while True: + row = snapshot(args.prometheus) + out.write(json.dumps(row, sort_keys=True) + "\n") + out.flush() + if time.monotonic() + args.interval > end: + break + time.sleep(args.interval) + print(f"saved {path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..99c65a2 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +for container in $(docker compose --env-file .env ps -q api); do + instance=$(docker exec "$container" python -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).headers['X-Instance-Id'])") + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop -t 5 "$container" >/dev/null + exit 0 + fi +done +echo "target API instance not found" >&2 +exit 1 diff --git a/src/api.py b/src/api.py new file mode 100644 index 0000000..69bc46a --- /dev/null +++ b/src/api.py @@ -0,0 +1,117 @@ +"""Messenger HTTP routes.""" + +import re +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response + +from .auth import basic_credentials, bearer_token +from .attachments import storage_ready, upload +from .attachment_read import download, get_view +from .models import CreateChat, CreateMessage, CreateUser +from .store import store + +router = APIRouter() + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +def ready(): + result = store.ready() + storage_ready() + return result + + +@router.post("/api/v1/users", status_code=201) +def register_user(body: CreateUser): + return store.register(body.username, body.display_name, body.password) + + +@router.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request, response: Response): + if await request.body(): + raise HTTPException(400) + username, password = basic_credentials(request.headers.get("authorization")) + response.headers["Cache-Control"] = "no-store" + return store.create_session(username, password) + + +def current_user(request: Request) -> dict: + return store.authenticated(bearer_token(request.headers.get("authorization"))) + + +def parse_limit(value: str) -> int: + try: + number = int(value) if value.isdecimal() else 0 + except ValueError: + raise HTTPException(400) from None + if not 1 <= number <= 100: + raise HTTPException(400) + return number + + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session(request: Request): + store.revoke_session(bearer_token(request.headers.get("authorization"))) + return Response(status_code=204) + + +@router.get("/api/v1/users/me") +def user_me(actor: dict = Depends(current_user)): + return actor + + +@router.post("/api/v1/chats", status_code=201) +def create_chat(body: CreateChat, actor: dict = Depends(current_user)): + return store.create_chat(actor["id"], body.title, body.member_ids) + + +@router.get("/api/v1/chats") +def list_chats( + limit: str = Query("50"), cursor: str | None = Query(None, min_length=1), actor: dict = Depends(current_user) +): + return store.list_chats(actor["id"], parse_limit(limit), cursor) + + +@router.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: UUID, actor: dict = Depends(current_user)): + return store.chat(actor["id"], str(chat_id)) + + +@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(chat_id: UUID, request: Request, filename: str = Query(...), actor: dict = Depends(current_user)): + return await upload(request, str(chat_id), actor["id"], filename) + + +@router.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: UUID, actor: dict = Depends(current_user)): + return get_view(actor["id"], attachment_id) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +def get_attachment_content( + attachment_id: UUID, variant: str = Query("original"), actor: dict = Depends(current_user) +): + return download(actor["id"], attachment_id, variant) + + +@router.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message(chat_id: UUID, body: CreateMessage, request: Request, actor: dict = Depends(current_user)): + key = request.headers.get("Idempotency-Key") + if key is None or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise HTTPException(400) + return store.create_message(actor["id"], str(chat_id), body.text, key, body.attachment_ids) + + +@router.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + chat_id: UUID, + limit: str = Query("50"), + cursor: str | None = Query(None, min_length=1), + actor: dict = Depends(current_user), +): + return store.list_messages(actor["id"], str(chat_id), parse_limit(limit), cursor) diff --git a/src/attachment_read.py b/src/attachment_read.py new file mode 100644 index 0000000..34f91a6 --- /dev/null +++ b/src/attachment_read.py @@ -0,0 +1,72 @@ +"""Authorize metadata and stream versioned variants from the private bucket.""" + +import logging +from uuid import UUID + +from fastapi import HTTPException +from fastapi.responses import StreamingResponse + +from .attachments import BUCKET, attachment_view, storage +from .store import connection + + +logger = logging.getLogger(__name__) + + +def metadata(actor_id: str, attachment_id: UUID) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a " + "JOIN memberships m ON m.chat_id=a.chat_id " + "WHERE a.id=%s AND m.user_id=%s", + (attachment_id, UUID(actor_id)), + ).fetchone() + if row is None: + raise HTTPException(404) + return row + + +def download(actor_id: str, attachment_id: UUID, variant: str) -> StreamingResponse: + row = metadata(actor_id, attachment_id) + if row["status"] != "ready": + raise HTTPException(409) + with connection() as conn: + selected = conn.execute( + "SELECT object_key,content_type,size_bytes FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s AND name=%s", + (attachment_id, row["pipeline_version"], variant), + ).fetchone() + if selected is None: + raise HTTPException(404) + try: + obj = storage().get_object(BUCKET, selected["object_key"]) + except Exception as exc: + logger.warning("S3 read failed for attachment %s (%s)", attachment_id, type(exc).__name__) + raise HTTPException(503) from None + + def chunks(): + try: + yield from obj.stream(64 * 1024) + finally: + obj.close() + obj.release_conn() + + return StreamingResponse( + chunks(), media_type=selected["content_type"], + headers={"Content-Length": str(selected["size_bytes"])} + ) + + +def get_view(actor_id: str, attachment_id: UUID) -> dict: + row = metadata(actor_id, attachment_id) + view = attachment_view(row) + if row["status"] == "ready": + with connection() as conn: + variants = conn.execute( + "SELECT name,content_type,size_bytes,width,height,sha256 " + "FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s " + "AND name <> 'original' ORDER BY name", + (attachment_id, row["pipeline_version"]), + ).fetchall() + view["variants"] = [dict(variant) for variant in variants] + return view diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..13cb080 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,182 @@ +"""Private S3 access shared by attachment routes and maintenance jobs.""" + +import asyncio +import hashlib +import logging +import os +import unicodedata +from functools import lru_cache +from pathlib import Path +from tempfile import SpooledTemporaryFile +from uuid import UUID, uuid4 + +from fastapi import HTTPException, Request +from minio import Minio +from minio.error import S3Error +from opentelemetry import trace +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from PIL import Image, UnidentifiedImageError +from urllib3 import PoolManager, Retry, Timeout + +from .pipeline_jobs import queue_attachment, renew_upload, reserve_upload +from .store import connection, store, timestamp + + +BUCKET = "messenger-attachments" +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +logger = logging.getLogger(__name__) +tracer = trace.get_tracer(__name__) +trace_context = TraceContextTextMapPropagator() + + +def secret(name: str) -> str: + return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip() + + +@lru_cache(maxsize=1) +def storage() -> Minio: + return Minio( + os.environ.get("S3_ENDPOINT", "s3:9000"), + access_key=secret("S3_ACCESS_KEY_ID"), + secret_key=secret("S3_SECRET_ACCESS_KEY"), + secure=False, + region="us-east-1", + http_client=PoolManager(timeout=Timeout(connect=2, read=10), retries=Retry(total=0)), + ) + + +def storage_ready() -> None: + try: + storage().stat_object(BUCKET, "health/ready-probe") + except S3Error as exc: + if exc.code != "NoSuchKey": + raise HTTPException(503) from None + except Exception: + raise HTTPException(503) from None + + +def attachment_view(row: dict) -> dict: + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), + "filename": row["filename"], "content_type": row["content_type"], + "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"], + "variants": [], "error_code": row["error_code"], + "created_at": timestamp(row["created_at"]), "updated_at": timestamp(row["updated_at"]), + } + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or filename in {".", ".."}: + raise HTTPException(400) + if any(char in "/\\" or unicodedata.category(char) == "Cc" for char in filename): + raise HTTPException(400) + + +def validate_image(stream, declared_type: str) -> str: + try: + with Image.open(stream) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) + if actual_type is None or actual_type != declared_type: + raise HTTPException(415) + if image.width * image.height > MAX_PIXELS: + raise HTTPException(413) + image.verify() + stream.seek(0) + with Image.open(stream) as image: + image.load() + return actual_type + except Image.DecompressionBombError: + raise HTTPException(413) from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise HTTPException(415) from None + + +def verify_staging(client: Minio, key: str, size: int, sha256: str, content_type: str) -> None: + """Confirm complete durable bytes; an S3 ETag is not a SHA-256 checksum.""" + info = client.stat_object(BUCKET, key) + if info.size != size or info.content_type.split(";", 1)[0] != content_type: + raise ValueError("staging metadata mismatch") + obj = client.get_object(BUCKET, key) + try: + digest = hashlib.sha256() + received = 0 + for chunk in obj.stream(64 * 1024): + received += len(chunk) + if received > size: + raise ValueError("staging size mismatch") + digest.update(chunk) + if received != size or digest.hexdigest() != sha256: + raise ValueError("staging checksum mismatch") + finally: + obj.close() + obj.release_conn() + + +def persist_attachment(chat_id: str, actor_id: str, filename: str, content_type: str, size: int, sha256: str, stream) -> dict: + attachment_id = uuid4() + staging_key = f"staging/{attachment_id}" + object_key = f"originals/{attachment_id}" + with connection() as conn: + reserve_upload(conn, attachment_id, staging_key) + + try: + stream.seek(0) + client = storage() + with tracer.start_as_current_span("s3.stage_upload", record_exception=False, set_status_on_exception=False): + client.put_object(BUCKET, staging_key, stream, size, content_type=content_type) + with connection() as conn: + if not renew_upload(conn, attachment_id): + raise HTTPException(503) + with tracer.start_as_current_span("s3.verify_staging", record_exception=False, set_status_on_exception=False): + verify_staging(client, staging_key, size, sha256, content_type) + carrier = {} + trace_context.inject(carrier) + with connection() as conn: + with tracer.start_as_current_span("db.queue_attachment", record_exception=False, + set_status_on_exception=False): + row, _job_id = queue_attachment( + conn, attachment_id=attachment_id, chat_id=UUID(chat_id), owner_id=UUID(actor_id), + filename=filename, staging_key=staging_key, object_key=object_key, + content_type=content_type, size_bytes=size, sha256=sha256, + traceparent=carrier.get("traceparent"), tracestate=carrier.get("tracestate"), + ) + except HTTPException: + raise + except Exception as exc: + # A lost COMMIT acknowledgement may leave a valid queued job. Keep the + # staging object; cleanup must check PostgreSQL before deleting it. + logger.warning("Attachment handoff failed for %s (%s)", attachment_id, type(exc).__name__) + raise HTTPException(503) from None + return attachment_view(row) + + +async def upload(request: Request, chat_id: str, actor_id: str, filename: str) -> dict: + await asyncio.to_thread(store.chat, actor_id, chat_id) + validate_filename(filename) + declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if declared_type not in {"image/png", "image/jpeg"}: + raise HTTPException(415) + length = request.headers.get("content-length") + if length is not None: + try: + if int(length) > MAX_BYTES: + raise HTTPException(413) + except ValueError: + raise HTTPException(400) from None + + with SpooledTemporaryFile(max_size=1024 * 1024) as stream: + digest = hashlib.sha256() + size = 0 + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise HTTPException(413) + stream.write(chunk) + digest.update(chunk) + stream.seek(0) + validate_image(stream, declared_type) + return await asyncio.to_thread( + persist_attachment, chat_id, actor_id, filename, declared_type, size, digest.hexdigest(), stream + ) diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 0000000..5714b17 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,43 @@ +"""Parse HTTP credentials without exposing them in errors or logs.""" + +import base64 +import binascii +import hashlib +import re + +from fastapi import HTTPException + + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +USERNAME = re.compile(r"[a-z0-9_]{3,32}\Z") + + +def unauthorized(challenge: str) -> HTTPException: + return HTTPException(401, headers={"WWW-Authenticate": challenge}) + + +def basic_credentials(header: str | None) -> tuple[str, str]: + if not header or not header.startswith("Basic "): + raise unauthorized(BASIC_CHALLENGE) + try: + raw = base64.b64decode(header[6:], validate=True).decode("utf-8") + username, password = raw.split(":", 1) + except (binascii.Error, UnicodeError, ValueError): + raise unauthorized(BASIC_CHALLENGE) from None + if not USERNAME.fullmatch(username) or not 12 <= len(password) <= 128: + raise unauthorized(BASIC_CHALLENGE) + return username, password + + +def bearer_token(header: str | None) -> str: + if not header or not header.startswith("Bearer "): + raise unauthorized(BEARER_CHALLENGE) + token = header[7:] + if not 32 <= len(token) <= 256 or not token.isascii() or not re.fullmatch(r"[A-Za-z0-9_-]+", token): + raise unauthorized(BEARER_CHALLENGE) + return token + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode("ascii")).digest() diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..48947b0 --- /dev/null +++ b/src/db.py @@ -0,0 +1,40 @@ +"""Bounded PostgreSQL pool shared by API storage and health checks.""" + +import os +import atexit +from functools import lru_cache +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from psycopg_pool import PoolTimeout + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get("DATABASE_URL", "postgresql:///messenger") + + +@lru_cache(maxsize=1) +def pool() -> ConnectionPool: + result = ConnectionPool( + database_url(), min_size=0, max_size=8, timeout=5, + kwargs={"connect_timeout": 5, "options": "-c statement_timeout=5000", "row_factory": dict_row}, + open=True, + ) + atexit.register(result.close) + from .telemetry import register_pool_metrics + register_pool_metrics(result) + return result + + +def ready() -> bool: + try: + with pool().connection() as conn: + conn.execute("SELECT 1 FROM users LIMIT 0") + return True + except (psycopg.Error, PoolTimeout, OSError): + return False diff --git a/src/image_variants.py b/src/image_variants.py new file mode 100644 index 0000000..72a8cd3 --- /dev/null +++ b/src/image_variants.py @@ -0,0 +1,83 @@ +"""Deterministic, bounded image variants for one accepted PNG or JPEG.""" + +import hashlib +from io import BytesIO + +from PIL import Image, ImageOps, UnidentifiedImageError + + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +class InvalidImage(ValueError): + pass + + +def _fit(image: Image.Image, bound: int) -> Image.Image: + factor = min(1, bound / max(image.size)) + size = (max(1, int(image.width * factor)), max(1, int(image.height * factor))) + return image.resize(size, Image.Resampling.LANCZOS) if size != image.size else image + + +def _jpeg(image: Image.Image) -> bytes: + output = BytesIO() + image.save(output, format="JPEG", quality=85, optimize=True) + return output.getvalue() + + +def _variant(data: bytes, content_type: str, image: Image.Image) -> dict: + return { + "data": data, + "content_type": content_type, + "size_bytes": len(data), + "width": image.width, + "height": image.height, + "sha256": hashlib.sha256(data).hexdigest(), + } + + +def render(data: bytes, content_type: str) -> dict[str, dict]: + """Preserve original bytes; orient, flatten alpha, strip metadata on derivatives.""" + if not 0 < len(data) <= MAX_BYTES or content_type not in {"image/png", "image/jpeg"}: + raise InvalidImage("invalid original") + try: + with Image.open(BytesIO(data)) as source: + if {"PNG": "image/png", "JPEG": "image/jpeg"}.get(source.format) != content_type: + raise InvalidImage("wrong image type") + if source.width * source.height > MAX_PIXELS: + raise InvalidImage("too many pixels") + source.load() + original_size = source.size + oriented = ImageOps.exif_transpose(source) + if oriented.mode in {"RGBA", "LA", "P"}: + rgba = oriented.convert("RGBA") + image = Image.new("RGB", rgba.size, "white") + image.paste(rgba, mask=rgba.getchannel("A")) + else: + image = oriented.convert("RGB") + + thumbnail = _fit(image, 256) + side = min(image.size) + left = (image.width - side) // 2 + top = (image.height - side) // 2 + crop = image.resize( + (128, 128), Image.Resampling.LANCZOS, + box=(left, top, left + side, top + side), + ) + optimized = _fit(image, 1280) + webp = BytesIO() + optimized.save(webp, format="WEBP", quality=82, method=4) + return { + "original": { + "data": data, "content_type": content_type, + "size_bytes": len(data), "width": original_size[0], "height": original_size[1], + "sha256": hashlib.sha256(data).hexdigest(), + }, + "thumbnail": _variant(_jpeg(thumbnail), "image/jpeg", thumbnail), + "crop": _variant(_jpeg(crop), "image/jpeg", crop), + "optimized": _variant(webp.getvalue(), "image/webp", optimized), + } + except (UnidentifiedImageError, OSError, SyntaxError, Image.DecompressionBombError) as exc: + raise InvalidImage("image decoding failed") from exc diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..d8a500c --- /dev/null +++ b/src/main.py @@ -0,0 +1,105 @@ +"""FastAPI entry point, contract responses, and safe HTTP telemetry.""" + +import logging +from time import monotonic +from uuid import uuid4 + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +from .api import router +from . import telemetry + +app = FastAPI() +app.include_router(router) +INSTANCE_ID = uuid4().hex +telemetry.configure("messenger-api") +HTTP_REQUESTS, HTTP_DURATION = telemetry.http_instruments() +TRACER = trace.get_tracer(__name__) +TRACE_CONTEXT = TraceContextTextMapPropagator() +logger = logging.getLogger(__name__) + +ERRORS = { + 400: ("invalid_request", "Неверный запрос"), + 401: ("unauthorized", "Требуется действительный пользователь"), + 404: ("not_found", "Объект не найден"), + 409: ("conflict", "Конфликт данных"), + 413: ("payload_too_large", "Слишком большой запрос"), + 415: ("unsupported_media_type", "Неподдерживаемый тип данных"), + 429: ("rate_limited", "Слишком много запросов"), + 503: ("unavailable", "Сервис недоступен"), +} + + +def error_response(request: Request, status: int, extra_headers: dict | None = None) -> JSONResponse: + code, message = ERRORS.get(status, ("internal_error", "Внутренняя ошибка")) + request_id = request.state.request_id + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID, **(extra_headers or {})}, + ) + + +@app.middleware("http") +async def request_contract(request: Request, call_next): + request.state.request_id = str(uuid4()) + started = monotonic() + status = 500 + context = TRACE_CONTEXT.extract(request.headers) + with TRACER.start_as_current_span("HTTP request", context=context, kind=SpanKind.SERVER, + record_exception=False, set_status_on_exception=False) as span: + try: + response = await _contract_response(request, call_next) + status = response.status_code + return response + finally: + route = getattr(request.scope.get("route"), "path", "unmatched") + status_class = f"{status // 100}xx" + attributes = {"method": request.method, "route": route, "status_class": status_class} + HTTP_REQUESTS.add(1, attributes) + HTTP_DURATION.record(monotonic() - started, attributes) + span.update_name(f"{request.method} {route}") + span.set_attribute("http.request.method", request.method) + span.set_attribute("http.route", route) + span.set_attribute("http.response.status_code", status) + span.set_attribute("request.id", request.state.request_id) + if status >= 500: + span.set_status(Status(StatusCode.ERROR)) + logger.info("http_request", extra={"event": "http_request", "request_id": request.state.request_id, + "method": request.method, "route": route, "status_class": status_class}) + + +async def _contract_response(request: Request, call_next): + is_upload = request.url.path.startswith("/api/v1/chats/") and request.url.path.endswith("/attachments") + if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions" and not is_upload: + media_type, _, charset = request.headers.get("content-type", "").lower().partition(";") + if media_type.strip() != "application/json" or (charset and charset.strip() != "charset=utf-8"): + return error_response(request, 415) + try: + (await request.body()).decode("utf-8") + except UnicodeDecodeError: + return error_response(request, 400) + response = await call_next(request) + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(RequestValidationError) +async def validation_error(request: Request, exc: RequestValidationError): + return error_response(request, 400) + + +@app.exception_handler(HTTPException) +async def http_error(request: Request, exc: HTTPException): + return error_response(request, exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def unexpected_error(request: Request, exc: Exception): + return error_response(request, 500) diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..06b16be --- /dev/null +++ b/src/models.py @@ -0,0 +1,68 @@ +"""Request bodies from the lab 1 contract.""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class StrictBody(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class CreateUser(StrictBody): + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def nonblank_name(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank display name") + return value + + @field_validator("password") + @classmethod + def nonblank_password(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank password") + return value + + +class CreateChat(StrictBody): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def nonblank_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank title") + return value + + @field_validator("member_ids") + @classmethod + def valid_members(cls, value: list[str]) -> list[str]: + members = [str(UUID(member_id)) for member_id in value] + if len(set(members)) != len(members): + raise ValueError("duplicate member") + return members + + +class CreateMessage(StrictBody): + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def valid_attachments(cls, value: list[str]) -> list[str]: + ids = [str(UUID(item)) for item in value] + if len(set(ids)) != len(ids): + raise ValueError("duplicate attachment") + return ids + + @model_validator(mode="after") + def text_or_attachment(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("message needs text or attachment") + return self diff --git a/src/pipeline_jobs.py b/src/pipeline_jobs.py new file mode 100644 index 0000000..4be6206 --- /dev/null +++ b/src/pipeline_jobs.py @@ -0,0 +1,284 @@ +"""Atomic PostgreSQL state changes for the attachment processing pipeline. + +Callers own the connection and must commit before replying 202 or acking AMQP. +S3 writes and RabbitMQ publication happen outside these transactions. +""" + +from datetime import timezone +from uuid import uuid4 + + +REQUIRED_VARIANTS = {"original", "thumbnail", "crop", "optimized"} + + +def job_payload(row): + """Broker message per jobs.schema.json: identifiers only, never file bytes.""" + payload = { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": row["pipeline_version"], + "attempt": row["attempt"], + "created_at": row["created_at"].astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), + } + for key in ("traceparent", "tracestate"): + if row.get(key): + payload[key] = row[key] + return payload + + +def reserve_upload(conn, attachment_id, staging_key, lease_seconds=300): + """Record an active staging PUT before it starts.""" + if lease_seconds < 1: + raise ValueError("invalid upload lease") + with conn.transaction(): + conn.execute( + "INSERT INTO attachment_uploads (id,staging_key,lease_expires_at) " + "VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s))", + (attachment_id, staging_key, lease_seconds), + ) + + +def renew_upload(conn, attachment_id, lease_seconds=300): + """Keep a slow, active upload protected from orphan cleanup.""" + if lease_seconds < 1: + raise ValueError("invalid upload lease") + with conn.transaction(): + return conn.execute( + "UPDATE attachment_uploads SET lease_expires_at=clock_timestamp()+make_interval(secs => %s) " + "WHERE id=%s AND lease_expires_at>clock_timestamp() RETURNING id", + (lease_seconds, attachment_id), + ).fetchone() is not None + + +def queue_attachment(conn, *, attachment_id, chat_id, owner_id, filename, + staging_key, object_key, content_type, size_bytes, sha256, + pipeline_version=1, traceparent=None, tracestate=None): + """Commit attachment, job, and outbox after the full S3 staging PUT.""" + job_id = uuid4() + with conn.transaction(): + marker = conn.execute( + "SELECT id FROM attachment_uploads WHERE id=%s AND staging_key=%s " + "AND lease_expires_at>clock_timestamp() FOR UPDATE", + (attachment_id, staging_key), + ).fetchone() + if marker is None: + raise ValueError("upload reservation is missing") + row = conn.execute( + "INSERT INTO attachments " + "(id,chat_id,owner_id,filename,object_key,staging_key,content_type,size_bytes,sha256,status,pipeline_version) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s) " + "RETURNING id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,error_code,created_at,updated_at", + (attachment_id, chat_id, owner_id, filename, object_key, staging_key, + content_type, size_bytes, sha256, pipeline_version), + ).fetchone() + conn.execute( + "INSERT INTO attachment_jobs (id,attachment_id,pipeline_version,traceparent,tracestate) " + "VALUES (%s,%s,%s,%s,%s)", + (job_id, attachment_id, pipeline_version, traceparent, tracestate), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)", + (job_id,), + ) + conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (attachment_id,)) + return row, job_id + + +def claim_outbox(conn, limit=10, lease_seconds=30): + """Lease pending publications; publisher confirms must call confirm_outbox.""" + if not 1 <= limit <= 100 or lease_seconds < 1: + raise ValueError("invalid outbox lease") + token = uuid4() + with conn.transaction(): + rows = conn.execute( + "WITH due AS (" + " SELECT id FROM attachment_outbox" + " WHERE published_at IS NULL AND available_at<=clock_timestamp()" + " AND (claim_expires_at IS NULL OR claim_expires_at %s)" + " FROM due,attachment_jobs j" + " WHERE o.id=due.id AND j.id=o.job_id" + " RETURNING o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at," + "j.traceparent,j.tracestate", + (limit, token, lease_seconds), + ).fetchall() + return token, rows + + +def confirm_outbox(conn, outbox_id, token): + """Mark published only after RabbitMQ publisher confirm.""" + with conn.transaction(): + return conn.execute( + "UPDATE attachment_outbox SET published_at=clock_timestamp()," + "claim_token=NULL,claim_expires_at=NULL " + "WHERE id=%s AND claim_token=%s AND published_at IS NULL RETURNING id", + (outbox_id, token), + ).fetchone() is not None + + +def claim_job(conn, job_id, delivery_attempt, lease_seconds=60): + """A stale/redelivered message cannot steal an active or finished job.""" + if lease_seconds < 1: + raise ValueError("invalid job lease") + token = uuid4() + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.staging_key,a.object_key,a.sha256,a.content_type,a.size_bytes," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j", + (job_id,), + ).fetchone() + if (job is None or job["status"] != "queued" or + job["pipeline_version"] != job["current_pipeline_version"] or + job["attempt"] != delivery_attempt or + job["next_attempt_at"] > conn.execute("SELECT clock_timestamp() AS now").fetchone()["now"]): + return None + conn.execute( + "UPDATE attachment_jobs SET status='processing',lease_token=%s," + "lease_expires_at=clock_timestamp()+make_interval(secs => %s)," + "updated_at=clock_timestamp() WHERE id=%s", + (token, lease_seconds, job_id), + ) + conn.execute( + "UPDATE attachments SET status='processing',updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s AND status='queued'", + (job["attachment_id"], job["pipeline_version"]), + ) + return {**job, "lease_token": token} + + +def _reschedule_locked(conn, job, *, max_attempts, delay_seconds, error_code, permanent): + if max_attempts < 1 or delay_seconds < 0: + raise ValueError("invalid retry configuration") + failed = permanent or job["attempt"] - job["replay_start_attempt"] + 1 >= max_attempts + if failed: + code = error_code if permanent else "retry_exhausted" + conn.execute( + "UPDATE attachment_jobs SET status='failed',lease_token=NULL,lease_expires_at=NULL," + "error_code=%s,updated_at=clock_timestamp() WHERE id=%s", + (code, job["id"]), + ) + conn.execute( + "UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (code, job["attachment_id"], job["pipeline_version"]), + ) + else: + attempt = job["attempt"] + 1 + due = conn.execute( + "UPDATE attachment_jobs SET status='queued',attempt=%s,lease_token=NULL,lease_expires_at=NULL," + "next_attempt_at=clock_timestamp()+make_interval(secs => %s)," + "error_code=%s,updated_at=clock_timestamp() WHERE id=%s RETURNING next_attempt_at", + (attempt, delay_seconds, error_code, job["id"]), + ).fetchone()["next_attempt_at"] + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt,available_at) VALUES (%s,%s,%s)", + (job["id"], attempt, due), + ) + conn.execute( + "UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (job["attachment_id"], job["pipeline_version"]), + ) + return "failed" if failed else "queued" + + +def finish_attempt(conn, job_id, token, *, max_attempts, delay_seconds=0, + error_code="processing_failed", permanent=False): + """Persist retry or terminal failure before the worker acks delivery.""" + if error_code not in {"invalid_image", "processing_failed", "storage_unavailable", "worker_timeout"}: + raise ValueError("unsafe error code") + with conn.transaction(): + job = conn.execute( + "SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job_id,) + ).fetchone() + if job is None or job["status"] != "processing" or job["lease_token"] != token: + raise ValueError("job lease is no longer owned") + return _reschedule_locked( + conn, job, max_attempts=max_attempts, delay_seconds=delay_seconds, + error_code=error_code, permanent=permanent, + ) + + +def recover_expired(conn, *, max_attempts, delay_seconds=0, limit=20): + """Requeue dead workers; old delivery attempts become harmless duplicates.""" + if not 1 <= limit <= 100: + raise ValueError("invalid recovery batch") + with conn.transaction(): + jobs = conn.execute( + "SELECT * FROM attachment_jobs WHERE status='processing' " + "AND lease_expires_at<=clock_timestamp() " + "ORDER BY lease_expires_at,id LIMIT %s FOR UPDATE SKIP LOCKED", + (limit,), + ).fetchall() + return [ + (job["id"], _reschedule_locked( + conn, job, max_attempts=max_attempts, delay_seconds=delay_seconds, + error_code="worker_timeout", permanent=False, + )) + for job in jobs + ] + + +def complete_job(conn, job_id, token, variants): + """Record every versioned effect, then expose ready atomically. + + Returns True when this version is still current. A duplicate ready job is + a harmless no-op; an old version never changes the current attachment. + """ + if set(variants) != REQUIRED_VARIANTS: + raise ValueError("incomplete variant set") + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.object_key,a.sha256,a.content_type,a.size_bytes," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j", + (job_id,), + ).fetchone() + if job is None: + raise ValueError("unknown job") + if job["status"] == "ready": + return job["pipeline_version"] == job["current_pipeline_version"] + if job["status"] != "processing" or job["lease_token"] != token: + raise ValueError("job lease is no longer owned") + original = variants["original"] + if (original["object_key"] != job["object_key"] or + original["sha256"] != job["sha256"] or + original["content_type"] != job["content_type"] or + original["size_bytes"] != job["size_bytes"]): + raise ValueError("original does not match accepted upload") + for name, value in variants.items(): + fields = (value["object_key"], value["content_type"], value["size_bytes"], + value["width"], value["height"], value["sha256"]) + conn.execute( + "INSERT INTO attachment_variants " + "(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,width,height,sha256) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING", + (job["attachment_id"], job["pipeline_version"], name, *fields), + ) + stored = conn.execute( + "SELECT object_key,content_type,size_bytes,width,height,sha256 " + "FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s", + (job["attachment_id"], job["pipeline_version"], name), + ).fetchone() + if stored is None or tuple(stored[key] for key in + ("object_key", "content_type", "size_bytes", "width", "height", "sha256")) != fields: + raise ValueError("versioned variant conflicts with a previous effect") + conn.execute( + "UPDATE attachment_jobs SET status='ready',lease_token=NULL,lease_expires_at=NULL," + "error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", + (job_id,), + ) + current = conn.execute( + "UPDATE attachments SET status='ready',error_code=NULL," + "updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s RETURNING id", + (job["attachment_id"], job["pipeline_version"]), + ).fetchone() + return current is not None diff --git a/src/publisher.py b/src/publisher.py new file mode 100644 index 0000000..02d2590 --- /dev/null +++ b/src/publisher.py @@ -0,0 +1,114 @@ +"""Publish PostgreSQL outbox jobs with RabbitMQ confirms.""" + +import json +import logging +import os +import time +from pathlib import Path + +import pika +import psycopg +from opentelemetry import metrics, trace +from opentelemetry.metrics import Observation +from opentelemetry.trace import SpanKind +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from psycopg_pool import PoolTimeout + +from .db import pool +from .pipeline_jobs import claim_outbox, confirm_outbox, job_payload +from .telemetry import configure + + +logger = logging.getLogger("src.publisher") +QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") +tracer = trace.get_tracer(__name__) +trace_context = TraceContextTextMapPropagator() + + +def register_queue_metrics(): + """Sample the durable backlog; broker delivery does not erase queued jobs.""" + meter = metrics.get_meter("messenger.queue") + + def backlog(_): + try: + with pool().connection() as conn: + row = conn.execute( + "SELECT count(*) AS depth," + "coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS oldest_age " + "FROM attachment_jobs WHERE status='queued'" + ).fetchone() + return row + except (psycopg.Error, PoolTimeout, OSError): + return None + + def observe(field): + def callback(options): + row = backlog(options) + return [Observation(row[field], {"queue_name": QUEUE})] if row is not None else [] + return callback + + meter.create_observable_gauge("messenger.queue.depth", callbacks=[observe("depth")]) + meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[observe("oldest_age")], unit="s") + + +def publish_batch(channel): + with pool().connection() as conn: + token, rows = claim_outbox(conn, limit=10, lease_seconds=30) + for row in rows: + parent = trace_context.extract({key: row[key] for key in ("traceparent", "tracestate") if row.get(key)}) + with tracer.start_as_current_span("attachments.publish", context=parent, kind=SpanKind.PRODUCER, + record_exception=False, set_status_on_exception=False) as span: + span.set_attribute("messaging.destination.name", QUEUE) + span.set_attribute("messaging.message.attempt", row["attempt"]) + carrier = {} + trace_context.inject(carrier) + body = json.dumps({**job_payload(row), **carrier}, separators=(",", ":")).encode("utf-8") + channel.basic_publish( + exchange="", routing_key=QUEUE, body=body, mandatory=True, + properties=pika.BasicProperties( + content_type="application/json", delivery_mode=2, + message_id=str(row["job_id"]), type="attachment.process.v1", + ), + ) + # basic_publish returns only after a broker confirm in confirm mode. + with pool().connection() as conn: + if not confirm_outbox(conn, row["id"], token): + logger.warning("Outbox claim expired before confirm for row %s", row["id"]) + logger.info("job_published", extra={"event": "job_published", "job_type": "attachment.process.v1", + "attempt": row["attempt"]}) + return len(rows) + + +def main(): + configure("messenger-publisher") + register_queue_metrics() + logging.basicConfig(level=logging.INFO) + logging.getLogger("pika").setLevel(logging.ERROR) + url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() + params = pika.URLParameters(url) + params.heartbeat = 30 + params.socket_timeout = 5 + params.blocked_connection_timeout = 10 + while True: + broker = None + try: + broker = pika.BlockingConnection(params) + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.confirm_delivery() + while broker.is_open: + if not publish_batch(channel): + broker.process_data_events(time_limit=1) + except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Publisher reconnecting after %s", type(exc).__name__) + time.sleep(2) + finally: + if broker is not None and broker.is_open: + try: + broker.close() + except (pika.exceptions.AMQPError, OSError): + pass + + +if __name__ == "__main__": + main() diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 0000000..96e7596 --- /dev/null +++ b/src/replay.py @@ -0,0 +1,64 @@ +"""Explicit, audited replay of one failed current-version job.""" + +import argparse +from uuid import UUID + +from .db import pool + + +def replay_failed(conn, job_id: UUID, operator: str, reason: str) -> int: + if not 1 <= len(operator) <= 80 or not 1 <= len(reason) <= 200: + raise ValueError("operator and reason are required") + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.status AS attachment_status," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j,a", + (job_id,), + ).fetchone() + if (job is None or job["status"] != "failed" or + job["pipeline_version"] != job["current_pipeline_version"] or + job["attachment_status"] != "failed"): + raise ValueError("only a failed current-version job can be replayed") + attempt = job["attempt"] + 1 + conn.execute( + "UPDATE attachment_jobs SET status='queued',attempt=%s,replay_start_attempt=%s," + "next_attempt_at=clock_timestamp(),error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s", + (attempt, attempt, job_id), + ) + conn.execute( + "UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (job["attachment_id"], job["pipeline_version"]), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,%s)", + (job_id, attempt), + ) + conn.execute( + "INSERT INTO attachment_job_replays " + "(job_id,previous_attempt,new_attempt,previous_error_code,operator_name,reason) " + "VALUES (%s,%s,%s,%s,%s,%s)", + (job_id, job["attempt"], attempt, job["error_code"], operator, reason), + ) + return attempt + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("job_id", type=UUID) + parser.add_argument("--operator", required=True) + parser.add_argument("--reason", required=True) + args = parser.parse_args() + try: + with pool().connection() as conn: + attempt = replay_failed(conn, args.job_id, args.operator, args.reason) + except ValueError as exc: + parser.error(str(exc)) + print(f"Queued job {args.job_id} attempt {attempt}") + + +if __name__ == "__main__": + main() diff --git a/src/reprocess.py b/src/reprocess.py new file mode 100644 index 0000000..f7f4354 --- /dev/null +++ b/src/reprocess.py @@ -0,0 +1,57 @@ +"""Start a new pipeline version from a verified, already ready original.""" + +import argparse +from uuid import UUID, uuid4 + +from .db import pool + + +def start_new_version(conn, attachment_id: UUID) -> tuple[int, UUID]: + with conn.transaction(): + row = conn.execute( + "SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,) + ).fetchone() + if row is None or row["status"] != "ready": + raise ValueError("only a ready attachment can start a new version") + original = conn.execute( + "SELECT object_key,size_bytes,sha256 FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s AND name='original'", + (attachment_id, row["pipeline_version"]), + ).fetchone() + if (original is None or original["object_key"] != row["object_key"] or + original["size_bytes"] != row["size_bytes"] or + original["sha256"] != row["sha256"]): + raise ValueError("verified original is missing") + version = row["pipeline_version"] + 1 + job_id = uuid4() + conn.execute( + "UPDATE attachments SET pipeline_version=%s,status='queued',error_code=NULL," + "updated_at=clock_timestamp() WHERE id=%s", + (version, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) " + "VALUES (%s,%s,%s)", + (job_id, attachment_id, version), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)", + (job_id,), + ) + return version, job_id + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("attachment_id", type=UUID) + args = parser.parse_args() + try: + with pool().connection() as conn: + version, job_id = start_new_version(conn, args.attachment_id) + except ValueError as exc: + parser.error(str(exc)) + print(f"Queued attachment {args.attachment_id} version {version} job {job_id}") + + +if __name__ == "__main__": + main() diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..20fb9c9 --- /dev/null +++ b/src/server.py @@ -0,0 +1,14 @@ +"""Run the ASGI worker as the container's main process.""" + +import os + +import uvicorn + + +if __name__ == "__main__": + uvicorn.run( + "src.main:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + workers=1, + ) diff --git a/src/staging_cleanup.py b/src/staging_cleanup.py new file mode 100644 index 0000000..72d70b3 --- /dev/null +++ b/src/staging_cleanup.py @@ -0,0 +1,104 @@ +"""Remove abandoned staging objects after a grace period or a committed result.""" + +import logging +import os +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from uuid import UUID + +from minio import Minio + +from .db import pool + + +logger = logging.getLogger(__name__) +BUCKET = "messenger-attachments" + + +def storage(): + def secret(name): + return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip() + + return Minio( + os.environ.get("S3_ENDPOINT", "s3:9000"), + access_key=secret("S3_MAINTENANCE_KEY_ID"), + secret_key=secret("S3_MAINTENANCE_SECRET_KEY"), + secure=False, + region="us-east-1", + ) + + +def clean_once(client, grace_seconds=3600): + """Return removed count; lock DB state through each S3 delete.""" + if grace_seconds < 1: + raise ValueError("invalid staging grace period") + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + removed = 0 + for obj in client.list_objects(BUCKET, prefix="staging/", recursive=True): + key = obj.object_name + suffix = key.removeprefix("staging/") + try: + if key != f"staging/{UUID(suffix)}": + continue + except ValueError: + continue + with pool().connection() as conn, conn.transaction(): + attachment = conn.execute( + "SELECT a.status,EXISTS(" + "SELECT 1 FROM attachment_variants v WHERE v.attachment_id=a.id " + "AND v.pipeline_version=a.pipeline_version AND v.name='original' " + "AND v.object_key=a.object_key) AS original_ready " + "FROM attachments a WHERE a.staging_key=%s FOR UPDATE OF a", + (key,), + ).fetchone() + if attachment: + if attachment["status"] != "ready" or not attachment["original_ready"]: + continue + else: + if obj.last_modified is None or obj.last_modified > cutoff: + continue + marker = conn.execute( + "SELECT lease_expires_at>clock_timestamp() AS active " + "FROM attachment_uploads WHERE staging_key=%s FOR UPDATE", + (key,), + ).fetchone() + if marker and marker["active"]: + continue + client.remove_object(BUCKET, key) + if not attachment: + conn.execute( + "DELETE FROM attachment_uploads WHERE staging_key=%s " + "AND lease_expires_at<=clock_timestamp()", + (key,), + ) + removed += 1 + with pool().connection() as conn, conn.transaction(): + conn.execute( + "DELETE FROM attachment_uploads u WHERE u.lease_expires_at<" + "clock_timestamp()-make_interval(secs => %s) " + "AND NOT EXISTS (SELECT 1 FROM attachments a WHERE a.staging_key=u.staging_key)", + (grace_seconds,), + ) + return removed + + +def main(): + logging.basicConfig(level=logging.INFO) + client = storage() + grace = int(os.environ.get("STAGING_GRACE_SECONDS", "3600")) + interval = int(os.environ.get("STAGING_CLEANUP_INTERVAL_SECONDS", "60")) + if interval < 1: + raise ValueError("invalid cleanup interval") + while True: + try: + removed = clean_once(client, grace) + if removed: + logger.info("Removed %s staging objects", removed) + except Exception as exc: + logger.warning("Staging cleanup retrying after %s", type(exc).__name__) + time.sleep(interval) + + +if __name__ == "__main__": + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 0000000..db6425e --- /dev/null +++ b/src/store.py @@ -0,0 +1,303 @@ +"""PostgreSQL-backed users, chats and message history.""" + +import base64 +import binascii +import hashlib +import json +import os +import secrets +from contextlib import contextmanager +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +import psycopg +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from fastapi import HTTPException +from opentelemetry import trace +from opentelemetry.trace import Status, StatusCode +from psycopg.errors import UniqueViolation +from psycopg_pool import PoolTimeout + +from . import db +from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, token_hash, unauthorized + +password_hasher = PasswordHasher() +tracer = trace.get_tracer(__name__) + + +def timestamp(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def encode_cursor(resource: str, item: dict) -> str: + value = [resource, item["created_at"], item["id"]] + return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).decode().rstrip("=") + + +def decode_cursor(cursor: str | None, resource: str) -> tuple[datetime, UUID] | None: + if cursor is None: + return None + try: + raw = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True) + value = json.loads(raw) + if not isinstance(value, list) or len(value) != 3 or value[0] != resource: + raise ValueError + date_text, item_id = value[1:] + if not isinstance(date_text, str) or not date_text.endswith("Z") or not isinstance(item_id, str): + raise ValueError + date = datetime.fromisoformat(date_text[:-1] + "+00:00") + if date.utcoffset().total_seconds() != 0: + raise ValueError + return date, UUID(item_id) + except (binascii.Error, UnicodeError, ValueError, TypeError, AttributeError): + raise HTTPException(400) from None + + +@contextmanager +def connection(): + with tracer.start_as_current_span("db.store", record_exception=False, set_status_on_exception=False) as span: + span.set_attribute("db.system.name", "postgresql") + try: + with db.pool().connection() as conn: + yield conn + except (psycopg.Error, PoolTimeout, OSError): + span.set_status(Status(StatusCode.ERROR)) + raise HTTPException(503) from None + + +def user_view(row: dict) -> dict: + return { + "id": str(row["id"]), "username": row["username"], + "display_name": row["display_name"], "created_at": timestamp(row["created_at"]), + } + + +def chat_view(row: dict) -> dict: + return { + "id": str(row["id"]), "title": row["title"], + "member_ids": [str(member_id) for member_id in row["member_ids"]], + "created_at": timestamp(row["created_at"]), + } + + +def message_view(row: dict) -> dict: + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), "text": row["text"].decode("utf-8"), + "created_at": timestamp(row["created_at"]), + "attachment_ids": [str(attachment_id) for attachment_id in row.get("attachment_ids", [])], + } + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM memberships m WHERE m.chat_id=c.id ORDER BY m.position) AS member_ids""" + +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id=m.id ORDER BY a.position) AS attachment_ids""" + + +class Store: + def ready(self) -> dict: + if not db.ready(): + raise HTTPException(503) + return {"status": "ok"} + + def register(self, username: str, display_name: str, password: str) -> dict: + password_hash = password_hasher.hash(password) + with connection() as conn: + try: + row = conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,%s,%s,%s) " + "RETURNING id,username,display_name,created_at", + (uuid4(), username, display_name, password_hash), + ).fetchone() + except UniqueViolation: + raise HTTPException(409) from None + return user_view(row) + + def user(self, user_id: str | None) -> dict: + if not user_id: + raise HTTPException(401) + try: + user_uuid = UUID(user_id) + except (ValueError, TypeError): + raise HTTPException(401) from None + with connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at FROM users WHERE id=%s", (user_uuid,) + ).fetchone() + if row is None: + raise HTTPException(401) + return user_view(row) + + def create_session(self, username: str, password: str) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s", + (username,), + ).fetchone() + try: + valid = row is not None and password_hasher.verify(row["password_hash"], password) + except (VerificationError, InvalidHashError): + valid = False + if not valid: + raise unauthorized(BASIC_CHALLENGE) + token = secrets.token_urlsafe(32) + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + expiry = conn.execute( + "INSERT INTO sessions (token_hash,user_id,expires_at) " + "VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s)) RETURNING expires_at", + (token_hash(token), row["id"], ttl), + ).fetchone()["expires_at"] + return {"token": token, "token_type": "Bearer", "expires_at": timestamp(expiry), "user": user_view(row)} + + def authenticated(self, token: str) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT u.id,u.username,u.display_name,u.created_at FROM sessions s " + "JOIN users u ON u.id=s.user_id " + "WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + return user_view(row) + + def revoke_session(self, token: str) -> None: + with connection() as conn: + row = conn.execute( + "DELETE FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp() RETURNING token_hash", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + + def create_chat(self, actor_id: str, title: str, member_ids: list[str]) -> dict: + if actor_id not in member_ids: + raise HTTPException(400) + members = [UUID(member_id) for member_id in member_ids] + with connection() as conn: + count = conn.execute("SELECT count(*) AS n FROM users WHERE id=ANY(%s)", (members,)).fetchone()["n"] + if count != len(members): + raise HTTPException(400) + row = conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING id,title,created_at", (uuid4(), title)).fetchone() + for position, member_id in enumerate(members): + conn.execute( + "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,%s)", + (row["id"], member_id, position), + ) + return chat_view({**row, "member_ids": members}) + + def chat(self, actor_id: str, chat_id: str) -> dict: + with connection() as conn: + row = conn.execute( + f"SELECT {CHAT_COLUMNS} FROM chats c " + "JOIN memberships access ON access.chat_id=c.id " + "WHERE access.user_id=%s AND c.id=%s", + (UUID(actor_id), UUID(chat_id)), + ).fetchone() + if row is None: + raise HTTPException(404) + return chat_view(row) + + def list_chats(self, actor_id: str, limit: int, cursor: str | None) -> dict: + after = decode_cursor(cursor, f"chats:{actor_id}") + query = (f"SELECT {CHAT_COLUMNS} FROM chats c " + "JOIN memberships access ON access.chat_id=c.id WHERE access.user_id=%s") + params = [UUID(actor_id)] + if after: + query += " AND (c.created_at,c.id)>(%s,%s)" + params.extend(after) + query += " ORDER BY c.created_at,c.id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + rows = conn.execute(query, params).fetchall() + items = [chat_view(row) for row in rows[:limit]] + return {"items": items, "next_cursor": encode_cursor(f"chats:{actor_id}", items[-1]) if len(rows) > limit else None} + + def create_message(self, actor_id: str, chat_id: str, text: str, key: str, attachment_ids: list[str] | None = None) -> dict: + ids = [UUID(item) for item in attachment_ids or []] + if len(ids) > 10 or len(set(ids)) != len(ids) or (not text.strip() and not ids): + raise HTTPException(400) + chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + request_hash = hashlib.sha256(json.dumps( + {"text": text, "attachment_ids": [str(item) for item in ids]}, ensure_ascii=False, sort_keys=True, + ).encode()).digest() + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) + ).fetchone() + if member is None: + raise HTTPException(404) + conn.execute( + "DELETE FROM message_replays WHERE user_id=%s AND chat_id=%s AND request_key=%s " + "AND created_at dict: + after = decode_cursor(cursor, f"messages:{chat_id}") + chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + query = f"SELECT {MESSAGE_COLUMNS} FROM messages m WHERE m.chat_id=%s" + params = [chat_uuid] + if after: + query += " AND (m.created_at,m.id)>(%s,%s)" + params.extend(after) + query += " ORDER BY m.created_at,m.id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) + ).fetchone() + if member is None: + raise HTTPException(404) + rows = conn.execute(query, params).fetchall() + items = [message_view(row) for row in rows[:limit]] + return {"items": items, "next_cursor": encode_cursor(f"messages:{chat_id}", items[-1]) if len(rows) > limit else None} + + +store = Store() diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..8d68d31 --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,128 @@ +"""Bounded, process-local OTLP setup and low-cardinality instruments.""" + +import logging +import os +import time +from functools import lru_cache +from uuid import uuid4 + +from opentelemetry import metrics, trace +from opentelemetry.metrics import Observation +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + + +HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10) +JOB_BUCKETS = (0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120) +READY_BUCKETS = (0.5, 1, 2, 5, 10, 30, 60, 120, 300) + + +@lru_cache(maxsize=1) +def configure(service_name: str) -> bool: + """Export asynchronously when Compose supplies a Collector endpoint.""" + endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "").rstrip("/") + if not endpoint: + return False + resource = Resource.create({ + "service.name": service_name, + "service.version": os.environ.get("APP_VERSION", "lab6"), + "service.instance.id": uuid4().hex, + "deployment.environment.name": "lab", + }) + + traces = TracerProvider(resource=resource) + traces.add_span_processor(BatchSpanProcessor( + OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces", timeout=2), + max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000, + )) + trace.set_tracer_provider(traces) + + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics", timeout=2), + export_interval_millis=5000, export_timeout_millis=2000, + ) + metrics.set_meter_provider(MeterProvider( + resource=resource, metric_readers=[reader], + views=[ + View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)), + View(instrument_name="messenger.jobs.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + View(instrument_name="messenger.attachments.ready_duration", aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)), + ], + )) + + logs = LoggerProvider(resource=resource) + logs.add_log_record_processor(BatchLogRecordProcessor( + OTLPLogExporter(endpoint=f"{endpoint}/v1/logs", timeout=2), + max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000, + )) + set_logger_provider(logs) + app_logger = logging.getLogger("src") + app_logger.setLevel(logging.INFO) + app_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logs)) + meter = metrics.get_meter("messenger.runtime") + meter.create_observable_counter("messenger.process.cpu.time", callbacks=[ + lambda _: [Observation(time.process_time())], + ], unit="s") + def rss(_): + with open("/proc/self/statm", encoding="ascii") as statm: + return [Observation(int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"))] + + meter.create_observable_gauge("messenger.process.rss", callbacks=[rss], unit="By") + return True + + +@lru_cache(maxsize=1) +def http_instruments(): + meter = metrics.get_meter("messenger.http") + return ( + meter.create_counter("messenger.http.requests", unit="1"), + meter.create_histogram("messenger.http.duration", unit="s"), + ) + + +@lru_cache(maxsize=1) +def job_instruments(): + meter = metrics.get_meter("messenger.jobs") + return ( + meter.create_counter("messenger.jobs.processed", unit="1"), + meter.create_histogram("messenger.jobs.duration", unit="s"), + ) + + +@lru_cache(maxsize=1) +def pipeline_instruments(): + meter = metrics.get_meter("messenger.pipeline") + return ( + meter.create_histogram("messenger.attachments.ready_duration", unit="s"), + meter.create_up_down_counter("messenger.worker.busy"), + ) + + +def register_worker_capacity(): + metrics.get_meter("messenger.pipeline").create_observable_gauge( + "messenger.worker.capacity", callbacks=[lambda _: [Observation(1)]], + ) + + +def register_pool_metrics(pool): + """Read local pool counters without querying the database.""" + meter = metrics.get_meter("messenger.pool") + + def observe(_): + stats = pool.get_stats() + return [Observation(stats["pool_size"] - stats["pool_available"])] + + meter.create_observable_gauge("messenger.db.pool.used", callbacks=[observe]) + meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[ + lambda _: [Observation(pool.get_stats()["requests_waiting"])], + ]) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..b51e8a0 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,273 @@ +"""At-least-once RabbitMQ consumer with versioned, idempotent S3 effects.""" + +import hashlib +import json +import logging +import os +import random +import re +import signal +import time +from contextlib import contextmanager +from datetime import datetime, timezone +from io import BytesIO +from pathlib import Path +from uuid import UUID + +import pika +import psycopg +from minio.error import S3Error +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from psycopg_pool import PoolTimeout + +from .attachments import BUCKET, storage +from .db import pool +from .image_variants import InvalidImage, MAX_BYTES, render +from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired +from .telemetry import configure, job_instruments, pipeline_instruments, register_worker_capacity + + +logger = logging.getLogger("src.worker") +QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") +MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) +LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) +tracer = trace.get_tracer(__name__) +trace_context = TraceContextTextMapPropagator() + + +def parse_message(body: bytes) -> dict: + if len(body) > 4096: + raise ValueError("oversized job message") + item = json.loads(body) + required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + if (not isinstance(item, dict) or not required <= item.keys() or + item.keys() - required - {"traceparent", "tracestate"} or + type(item["schema_version"]) is not int or item["schema_version"] != 1 or + item["type"] != "attachment.process.v1" or + type(item["pipeline_version"]) is not int or item["pipeline_version"] < 1 or + type(item["attempt"]) is not int or item["attempt"] < 1 or + not isinstance(item["job_id"], str) or + not isinstance(item["attachment_id"], str) or + not isinstance(item["created_at"], str) or + ("traceparent" in item and + (not isinstance(item["traceparent"], str) or + re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", item["traceparent"]) is None)) or + ("tracestate" in item and + (not isinstance(item["tracestate"], str) or len(item["tracestate"]) > 512))): + raise ValueError("invalid job message") + UUID(item["job_id"]) + UUID(item["attachment_id"]) + created_at = datetime.fromisoformat(item["created_at"].replace("Z", "+00:00")) + if created_at.utcoffset() is None: + raise ValueError("job timestamp lacks timezone") + return item + + +def _source_bytes(job) -> bytes: + if not 0 < job["size_bytes"] <= MAX_BYTES: + raise InvalidImage("invalid accepted size") + # Reprocessing uses the byte-identical final original, so ready staging + # can be reclaimed without making a future pipeline version impossible. + key = job["staging_key"] if job["pipeline_version"] == 1 else job["object_key"] + response = storage().get_object(BUCKET, key) + try: + chunks = [] + size = 0 + digest = hashlib.sha256() + for chunk in response.stream(64 * 1024): + size += len(chunk) + if size > job["size_bytes"]: + raise InvalidImage("source exceeds accepted size") + chunks.append(chunk) + digest.update(chunk) + if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]: + raise InvalidImage("source checksum mismatch") + return b"".join(chunks) + finally: + response.close() + response.release_conn() + + +def _write_variants(job, variants: dict) -> dict: + extension = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"} + stored = {} + for name, variant in variants.items(): + key = (job["object_key"] if name == "original" else + f"variants/{job['attachment_id']}/v{job['pipeline_version']}/{name}.{extension[name]}") + data = variant["data"] + storage().put_object( + BUCKET, key, BytesIO(data), len(data), content_type=variant["content_type"] + ) + stored[name] = {key_name: value for key_name, value in variant.items() if key_name != "data"} + stored[name]["object_key"] = key + return stored + + +def _delay(attempt: int) -> float: + base = min(30, 2 ** min(attempt - 1, 5)) + return base + random.uniform(0, base / 2) + + +def _fault(point: str, job_id) -> None: + """Test-only process death at a named durable boundary for one job.""" + if (os.environ.get("LAB5_FAULT_POINT") == point and + os.environ.get("LAB5_FAULT_JOB_ID") == str(job_id)): + os._exit(72) + + +@contextmanager +def _deadline(seconds: int): + def expired(_signal, _frame): + raise TimeoutError("image job exceeded worker deadline") + + previous = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def process_message(item: dict) -> str: + with pool().connection() as conn: + job = claim_job(conn, UUID(item["job_id"]), item["attempt"], lease_seconds=LEASE_SECONDS) + if job is None: + return "duplicate_or_stale" + token = job["lease_token"] + _fault("after_claim_before_s3", job["id"]) + try: + with _deadline(LEASE_SECONDS): + with tracer.start_as_current_span("s3.read_source", record_exception=False, set_status_on_exception=False): + data = _source_bytes(job) + with tracer.start_as_current_span("image.render", record_exception=False, set_status_on_exception=False): + variants = render(data, job["content_type"]) + with tracer.start_as_current_span("s3.write_variants", record_exception=False, set_status_on_exception=False): + stored = _write_variants(job, variants) + _fault("after_s3_before_result_commit", job["id"]) + with pool().connection() as conn: + with tracer.start_as_current_span("db.complete_job", record_exception=False, set_status_on_exception=False): + current = complete_job(conn, job["id"], token, stored) + _fault("after_result_commit_before_ack", job["id"]) + if current: + ready_duration, _ = pipeline_instruments() + ready_duration.record(max(0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()), + {"job_type": "attachment.process.v1"}) + if current and job["staging_key"]: + try: + storage().remove_object(BUCKET, job["staging_key"]) + except Exception as exc: + # Periodic cleanup recovers this post-COMMIT gap. + logger.warning("Staging cleanup deferred for %s (%s)", job["id"], type(exc).__name__) + return "ready" + except psycopg.Error: + raise + except Exception as exc: + if isinstance(exc, InvalidImage): + code, permanent = "invalid_image", True + elif isinstance(exc, TimeoutError): + code, permanent = "worker_timeout", False + elif isinstance(exc, S3Error) or isinstance(exc, OSError): + code, permanent = "storage_unavailable", False + elif isinstance(exc, ValueError): + code, permanent = "processing_failed", True + else: + code, permanent = "processing_failed", False + logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code) + trace.get_current_span().add_event("job_retry_or_failure", {"error.code": code}) + with pool().connection() as conn: + try: + return finish_attempt( + conn, job["id"], token, max_attempts=MAX_ATTEMPTS, + delay_seconds=_delay(job["attempt"]), error_code=code, + permanent=permanent, + ) + except ValueError: + # A lease recovery already owns the next attempt. + return "stale_lease" + + +def traced_process_message(item: dict) -> str: + parent = trace_context.extract({key: item[key] for key in ("traceparent", "tracestate") if key in item}) + with tracer.start_as_current_span("attachments.process", context=parent, kind=SpanKind.CONSUMER, + record_exception=False, set_status_on_exception=False) as span: + span.set_attribute("messaging.destination.name", QUEUE) + span.set_attribute("messaging.message.attempt", item["attempt"]) + span.set_attribute("job.id", item["job_id"]) + span.set_attribute("attachment.id", item["attachment_id"]) + started = time.monotonic() + _, busy = pipeline_instruments() + busy.add(1) + outcome = "error" + try: + outcome = process_message(item) + return outcome + except (psycopg.Error, PoolTimeout, OSError): + outcome = "database_unavailable" + raise + finally: + busy.add(-1) + attributes = {"job_type": item["type"], "outcome": outcome} + processed, duration = job_instruments() + processed.add(1, attributes) + duration.record(time.monotonic() - started, attributes) + span.set_attribute("job.outcome", outcome) + if outcome in {"failed", "database_unavailable", "error"}: + span.set_status(Status(StatusCode.ERROR)) + logger.info("job_processed", extra={"event": "job_processed", "job_type": item["type"], + "outcome": outcome, "attempt": item["attempt"]}) + + +def main() -> None: + configure("messenger-worker") + register_worker_capacity() + logging.basicConfig(level=logging.INFO) + if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1: + raise SystemExit("invalid worker retry settings") + url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() + params = pika.URLParameters(url) + params.heartbeat = max(120, LEASE_SECONDS * 2) + params.socket_timeout = 5 + params.blocked_connection_timeout = 10 + while True: + broker = None + try: + broker = pika.BlockingConnection(params) + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.basic_qos(prefetch_count=1) + last_recovery = 0.0 + while broker.is_open: + if time.monotonic() - last_recovery >= 5: + with pool().connection() as conn: + recover_expired(conn, max_attempts=MAX_ATTEMPTS) + last_recovery = time.monotonic() + method, _, body = channel.basic_get(queue=QUEUE, auto_ack=False) + if method is None: + broker.process_data_events(time_limit=1) + continue + try: + item = parse_message(body) + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + logger.warning("Rejecting invalid job message") + channel.basic_reject(method.delivery_tag, requeue=False) + continue + try: + traced_process_message(item) + channel.basic_ack(method.delivery_tag) + except (psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__) + channel.basic_nack(method.delivery_tag, requeue=True) + time.sleep(2) + except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Worker reconnecting after %s", type(exc).__name__) + time.sleep(2) + finally: + if broker is not None and broker.is_open: + broker.close() + + +if __name__ == "__main__": + main() diff --git a/student-tests/publish_gap_inner.py b/student-tests/publish_gap_inner.py new file mode 100644 index 0000000..b27d6a5 --- /dev/null +++ b/student-tests/publish_gap_inner.py @@ -0,0 +1,65 @@ +"""Run in a one-shot publisher container to inject publish-before-confirm gap.""" + +import json +import os +import time +from pathlib import Path +from uuid import UUID, uuid4 + +import pika + +from src.db import pool +from src.publisher import QUEUE, job_payload, publish_batch + + +attachment_id = UUID(os.environ["FAULT_ATTACHMENT_ID"]) +url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() +broker = pika.BlockingConnection(pika.URLParameters(url)) +try: + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.confirm_delivery() + token = uuid4() + with pool().connection() as conn: + row = conn.execute( + "SELECT o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at " + "FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id " + "WHERE j.attachment_id=%s AND j.pipeline_version=1 AND o.attempt=1", + (attachment_id,), + ).fetchone() + assert row is not None + claimed = conn.execute( + "UPDATE attachment_outbox SET claim_token=%s," + "claim_expires_at=clock_timestamp()+interval '30 seconds' " + "WHERE id=%s AND claim_token IS NULL AND published_at IS NULL RETURNING id", + (token, row["id"]), + ).fetchone() + assert claimed is not None + body = json.dumps(job_payload(row), separators=(",", ":")).encode() + channel.basic_publish( + exchange="", routing_key=QUEUE, mandatory=True, body=body, + properties=pika.BasicProperties(content_type="application/json", delivery_mode=2), + ) + # Simulate process death after broker confirm, before PostgreSQL confirm. + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_outbox SET claim_expires_at=clock_timestamp()-interval '1 second' " + "WHERE id=%s AND claim_token=%s AND published_at IS NULL", + (row["id"], token), + ) + published = publish_batch(channel) + with pool().connection() as conn: + outbox = conn.execute( + "SELECT published_at IS NOT NULL AS confirmed FROM attachment_outbox " + "WHERE id=%s", (row["id"],), + ).fetchone() + deadline = time.monotonic() + 3 + while True: + count = channel.queue_declare(queue=QUEUE, passive=True).method.message_count + if count >= 2 or time.monotonic() >= deadline: + break + time.sleep(0.05) + assert published >= 1 and outbox["confirmed"] and count >= 2 + print(json.dumps({"confirmed_after_republish": True, "queued_copies_before_worker": count})) +finally: + broker.close() diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py new file mode 100644 index 0000000..404b863 --- /dev/null +++ b/student-tests/test_image_pipeline.py @@ -0,0 +1,74 @@ +"""Small checks for the worker's deterministic image and payload boundary.""" + +import json +import unittest +from io import BytesIO + +from PIL import Image + +from src.image_variants import InvalidImage, render +from src.worker import parse_message + + +def encoded(image, format="PNG", **options): + stream = BytesIO() + image.save(stream, format=format, **options) + return stream.getvalue() + + +class ImagePipelineTest(unittest.TestCase): + def test_contract_sizes_original_and_content_types(self): + original = encoded(Image.new("RGB", (320, 200), "red")) + result = render(original, "image/png") + self.assertEqual(result["original"]["data"], original) + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (256, 160)) + self.assertEqual((result["crop"]["width"], result["crop"]["height"]), (128, 128)) + self.assertEqual((result["optimized"]["width"], result["optimized"]["height"]), (320, 200)) + self.assertEqual([result[name]["content_type"] for name in ("thumbnail", "crop", "optimized")], + ["image/jpeg", "image/jpeg", "image/webp"]) + + def test_alpha_is_white_and_small_image_is_not_upscaled(self): + original = encoded(Image.new("RGBA", (24, 12), (255, 0, 0, 0))) + result = render(original, "image/png") + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (24, 12)) + with Image.open(BytesIO(result["thumbnail"]["data"])) as thumb: + self.assertTrue(all(channel >= 245 for channel in thumb.getpixel((0, 0)))) + + def test_exif_orientation_and_center_crop(self): + image = Image.new("RGB", (30, 10), "green") + image.paste("red", (0, 0, 10, 10)) + image.paste("blue", (20, 0, 30, 10)) + exif = Image.Exif() + exif[274] = 6 + original = encoded(image, "JPEG", exif=exif) + result = render(original, "image/jpeg") + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (10, 30)) + with Image.open(BytesIO(result["crop"]["data"])) as crop: + red, green, blue = crop.getpixel((64, 64)) + self.assertGreater(green, red) + self.assertGreater(green, blue) + + def test_rejects_wrong_type_and_corrupt_file(self): + original = encoded(Image.new("RGB", (2, 2), "black")) + with self.assertRaises(InvalidImage): + render(original, "image/jpeg") + with self.assertRaises(InvalidImage): + render(b"broken", "image/png") + + def test_payload_contains_only_versioned_job_identifiers(self): + body = { + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111", + "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", + "pipeline_version": 1, + "attempt": 1, + "created_at": "2026-09-01T12:00:00Z", + } + self.assertEqual(parse_message(json.dumps(body).encode()), body) + with self.assertRaises(ValueError): + parse_message(json.dumps({**body, "staging_key": "private"}).encode()) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_message_unicode.py b/student-tests/test_message_unicode.py new file mode 100644 index 0000000..4ff79d8 --- /dev/null +++ b/student-tests/test_message_unicode.py @@ -0,0 +1,156 @@ +"""Run with DATABASE_URL pointing to a disposable PostgreSQL database. + +Checks real HTTP writes, persisted history, retries, and migration of old rows. +""" + +import base64 +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.parse +import urllib.request +from uuid import uuid4 + +import psycopg +from psycopg.conninfo import make_conninfo + +ROOT = Path(__file__).resolve().parents[1] +LAB = json.loads((ROOT / "lab.json").read_text())["number"] +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database") +class MessageUnicodeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + cls.port = sock.getsockname()[1] + cls.start_server() + cls.addClassCleanup(cls.stop_server) + + @classmethod + def start_server(cls): + cls.server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"}, + ) + for _ in range(100): + if cls.server.poll() is not None: + raise AssertionError("API exited during startup") + try: + cls.call("GET", "/health/live") + return + except (AssertionError, urllib.error.URLError): + time.sleep(0.05) + cls.stop_server() + raise AssertionError("API did not become ready") + + @classmethod + def stop_server(cls): + cls.server.terminate() + try: + cls.server.wait(timeout=5) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait(timeout=5) + + @classmethod + def call(cls, method, path, body=None, headers=None, expected=200): + request = urllib.request.Request( + f"http://127.0.0.1:{cls.port}{path}", method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json", **(headers or {})}, + ) + try: + response = urllib.request.urlopen(request, timeout=10) + except urllib.error.HTTPError as exc: + response = exc + with response: + payload = response.read() + assert response.status == expected, (method, path, response.status, expected, payload) + return json.loads(payload) if payload else None + + def test_http_round_trip_after_restart_and_replay(self): + username, password = "u_" + uuid4().hex[:16], "Test-password-123" + user = self.call("POST", "/api/v1/users", { + "username": username, "display_name": "Unicode", "password": password, + }, expected=201) + session = self.call("POST", "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(), + }, expected=201) + auth = {"Authorization": "Bearer " + session["token"]} + chat = self.call("POST", "/api/v1/chats", { + "title": "Unicode", "member_ids": [user["id"]], + }, auth, 201) + path = f"/api/v1/chats/{chat['id']}/messages" + texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000] + messages, keys = [], [] + for text in texts: + key = uuid4().hex + message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(message["text"], text) + messages.append(message) + keys.append(key) + for text in ("", "\0" * 4001, "🙂" * 4001): + self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400) + self.stop_server() + self.start_server() + found, cursor = [], None + while True: + page = self.call("GET", path + "?limit=1" + ( + "&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else "" + ), headers=auth) + found.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(found, messages) + if LAB >= 3: + for text, message, key in zip(texts, messages, keys): + replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(replay, message) + self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409) + self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages)) + + def test_upgrade_preserves_old_text_and_is_repeatable(self): + schema = "unicode_" + uuid4().hex + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'CREATE SCHEMA "{schema}"') + isolated = make_conninfo(URL, options=f"-c search_path={schema}") + try: + old_text = "Старое 🙂 \\ literal \\u0000" + with psycopg.connect(isolated) as conn: + conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)") + for migration in sorted((ROOT / "migrations").glob("*.sql")): + version = int(migration.name.split("_", 1)[0]) + if version >= 9: + continue + conn.execute(migration.read_text()) + conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,)) + user, chat, message = uuid4(), uuid4(), uuid4() + conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,)) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,)) + conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user)) + conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text)) + for _ in range(2): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with psycopg.connect(isolated) as conn: + conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text()) + saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone() + self.assertEqual(saved.decode("utf-8"), old_text) + finally: + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'DROP SCHEMA "{schema}" CASCADE') + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_observability_failure.py b/student-tests/test_observability_failure.py new file mode 100644 index 0000000..dd49af8 --- /dev/null +++ b/student-tests/test_observability_failure.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Prove business API survives a bounded outage of all telemetry stores.""" + +import json +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlencode +from urllib.request import urlopen + + +ROOT = Path(__file__).resolve().parents[1] +FILES = ["-f", "compose.yaml"] +if (ROOT / "compose.local.yaml").exists(): + FILES += ["-f", "compose.local.yaml"] +FILES += ["-f", "compose.telemetry.yaml"] +COMPOSE = ["docker", "compose", *FILES, "--env-file", ".env"] +COUNTERS = ( + "otelcol_exporter_enqueue_failed_spans", + "otelcol_exporter_send_failed_spans", + "otelcol_exporter_send_failed_log_records", + "otelcol_exporter_send_failed_metric_points", + "otelcol_receiver_refused_spans", + "otelcol_receiver_refused_log_records", + "otelcol_receiver_refused_metric_points", +) + + +def run(*args): + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True) + + +def stats(): + result = run("docker", "stats", "--no-stream", "--format", "{{json .}}", + "mtusi2026lab6-otel-collector-1", "mtusi2026lab6-api-1", "mtusi2026lab6-worker-1") + return [json.loads(line) for line in result.stdout.splitlines()] if result.returncode == 0 else [] + + +def metric(name): + url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": f"sum({name})"}) + try: + with urlopen(url, timeout=5) as response: + rows = json.load(response)["data"]["result"] + except (OSError, ValueError): + return None + return float(rows[0]["value"][1]) if rows else None + + +def stores_up(): + url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": "up{job=~\"prometheus|loki|tempo\"}"}) + try: + with urlopen(url, timeout=5) as response: + rows = json.load(response)["data"]["result"] + except (OSError, ValueError): + return False + return len(rows) == 3 and all(row["value"][1] == "1" for row in rows) + + +def main(): + output = ROOT / "evidence/telemetry-backend-outage-2026-10-02.json" + data = {"started": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "stores": ["prometheus", "loki", "tempo"], "stats_before": stats(), + "collector_counters_before": {name: metric(name) for name in COUNTERS}} + stopped = False + try: + result = run(*COMPOSE, "stop", "prometheus", "loki", "tempo") + if result.returncode: + raise RuntimeError(result.stderr[-500:]) + stopped = True + data["stores_stopped"] = datetime.now(timezone.utc).isoformat(timespec="seconds") + smoke = run("make", "test", "BASE_URL=http://127.0.0.1:8080") + data["smoke_passed"] = smoke.returncode == 0 + data["smoke_tail"] = (smoke.stdout + smoke.stderr).splitlines()[-5:] + load = run("python3", "tests/load.py", "--base-url", "http://127.0.0.1:8080", + "--duration", "30", "--concurrency", "4", + "--output", "evidence/telemetry-outage-http-load-2026-10-02.json") + data["load_passed"] = load.returncode == 0 + data["stats_during"] = stats() + # Longer than Collector's 30-second exporter retry horizon. + time.sleep(35) + data["stats_after_35s"] = stats() + finally: + if stopped: + restored = run(*COMPOSE, "up", "-d", "prometheus", "loki", "tempo") + data["stores_restored"] = restored.returncode == 0 + data["stores_restored_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds") + healthy_since = None + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + if stores_up(): + healthy_since = healthy_since or time.monotonic() + if time.monotonic() - healthy_since >= 30: + break + else: + healthy_since = None + time.sleep(5) + data["stores_healthy"] = (healthy_since is not None and + time.monotonic() - healthy_since >= 30 and stores_up()) + data["collector_counters_after"] = {name: metric(name) for name in COUNTERS} + data["collector_counter_delta"] = { + name: (data["collector_counters_after"][name] - data["collector_counters_before"][name] + if data["collector_counters_before"][name] is not None and + data["collector_counters_after"][name] is not None else None) + for name in COUNTERS + } + output.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"smoke_passed": data.get("smoke_passed"), + "load_passed": data.get("load_passed"), + "stores_restored": data.get("stores_restored"), + "collector_counter_delta": data["collector_counter_delta"], + "output": str(output)}, sort_keys=True)) + if not all(data.get(key) for key in ("smoke_passed", "load_passed", "stores_restored", "stores_healthy")): + raise SystemExit("business path or telemetry recovery failed") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_outbox_payload.py b/student-tests/test_outbox_payload.py new file mode 100644 index 0000000..b1c37de --- /dev/null +++ b/student-tests/test_outbox_payload.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Keep broker messages within the public job schema and free of file data.""" + +import json +import sys +import unittest +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "tests")) +from client import validate # noqa: E402 +from src.pipeline_jobs import job_payload # noqa: E402 + + +class OutboxPayloadTest(unittest.TestCase): + def test_job_message_has_only_contract_fields(self): + schema = json.loads((ROOT / "contracts/jobs.schema.json").read_text(encoding="utf-8")) + row = { + "job_id": uuid4(), "attachment_id": uuid4(), "pipeline_version": 2, + "attempt": 3, "created_at": datetime.now(timezone.utc), + "object_key": "staging/must-not-leak", "bytes": b"must-not-leak", + "traceparent": "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + "tracestate": "lab=upload", + } + message = job_payload(row) + validate(message, schema) + self.assertEqual(set(message), set(schema["required"]) | {"traceparent", "tracestate"}) + self.assertEqual(message["traceparent"], row["traceparent"]) + self.assertEqual(message["tracestate"], row["tracestate"]) + self.assertNotIn("staging", json.dumps(message)) + self.assertNotIn("bytes", json.dumps(message)) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_pipeline_faults.py b/student-tests/test_pipeline_faults.py new file mode 100644 index 0000000..63a8e8e --- /dev/null +++ b/student-tests/test_pipeline_faults.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Deterministic fault windows for the versioned attachment pipeline. + +Run in a one-shot worker image while the normal worker and publisher are stopped. +All rows and objects are synthetic and stay in the educational volumes. +""" + +import hashlib +import json +import os +import subprocess +import sys +import time +from io import BytesIO +from uuid import uuid4 + +from PIL import Image + +from src.attachments import BUCKET, storage +from src.db import pool +from src.pipeline_jobs import ( + claim_job, finish_attempt, queue_attachment, recover_expired, reserve_upload, +) +from src.replay import replay_failed +from src.reprocess import start_new_version +from src.worker import process_message + + +image = BytesIO() +Image.new("RGB", (320, 200), "#2765a8").save(image, format="PNG") +DATA = image.getvalue() +SHA = hashlib.sha256(DATA).hexdigest() + + +def create_case(): + user_id, chat_id, attachment_id = uuid4(), uuid4(), uuid4() + staging_key = f"staging/{attachment_id}" + with pool().connection() as conn: + with conn.transaction(): + conn.execute( + "INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)", + (user_id, f"fault_{user_id.hex[:20]}", "Pipeline fault fixture", "synthetic"), + ) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s)", (chat_id, "Pipeline fault fixture")) + conn.execute( + "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", + (chat_id, user_id), + ) + reserve_upload(conn, attachment_id, staging_key) + storage().put_object(BUCKET, staging_key, BytesIO(DATA), len(DATA), content_type="image/png") + with pool().connection() as conn: + _, job_id = queue_attachment( + conn, attachment_id=attachment_id, chat_id=chat_id, owner_id=user_id, + filename="fixture.png", staging_key=staging_key, + object_key=f"originals/{attachment_id}", content_type="image/png", + size_bytes=len(DATA), sha256=SHA, + ) + return attachment_id, job_id + + +def item(job_id, attempt=1): + return {"job_id": str(job_id), "attempt": attempt} + + +def state(attachment_id, version): + with pool().connection() as conn: + attachment = conn.execute( + "SELECT status,pipeline_version,sha256 FROM attachments WHERE id=%s", (attachment_id,) + ).fetchone() + rows = conn.execute( + "SELECT name,sha256 FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name", + (attachment_id, version), + ).fetchall() + return attachment, {row["name"]: row["sha256"] for row in rows} + + +def assert_ready(attachment_id, version): + attachment, variants = state(attachment_id, version) + assert attachment["status"] == "ready" and attachment["pipeline_version"] == version + assert attachment["sha256"] == SHA + assert set(variants) == {"original", "thumbnail", "crop", "optimized"} + assert variants["original"] == SHA + return variants + + +def expire_and_recover(job_id): + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_jobs SET lease_expires_at=clock_timestamp()-interval '1 second' " + "WHERE id=%s AND status='processing'", + (job_id,), + ) + recovered = recover_expired(conn, max_attempts=5) + assert (job_id, "queued") in recovered + + +def crash(job_id, point): + child = subprocess.run( + [sys.executable, "-c", "from src.worker import process_message; " + f"process_message({{'job_id': '{job_id}', 'attempt': 1}})"], + env={**os.environ, "LAB5_FAULT_POINT": point, "LAB5_FAULT_JOB_ID": str(job_id)}, + capture_output=True, text=True, + ) + assert child.returncode == 72, (point, child.returncode, child.stderr) + + +def main(): + results = {} + + attachment, job = create_case() + crash(job, "after_result_commit_before_ack") + first = assert_ready(attachment, 1) + assert process_message(item(job)) == "duplicate_or_stale" + assert assert_ready(attachment, 1) == first + results["after_result_commit_before_ack_duplicate"] = "pass" + + attachment, job = create_case() + crash(job, "after_claim_before_s3") + expire_and_recover(job) + assert process_message(item(job, 2)) == "ready" + assert_ready(attachment, 1) + results["before_side_effect_lease_recovery"] = "pass" + + attachment, job = create_case() + crash(job, "after_s3_before_result_commit") + expire_and_recover(job) + assert process_message(item(job, 2)) == "ready" + assert_ready(attachment, 1) + results["after_s3_before_result_commit"] = "pass" + + attachment, old_job = create_case() + assert process_message(item(old_job)) == "ready" + with pool().connection() as conn: + version, new_job = start_new_version(conn, attachment) + assert version == 2 + assert process_message(item(old_job)) == "duplicate_or_stale" + assert process_message(item(new_job)) == "ready" + newer = assert_ready(attachment, 2) + assert process_message(item(old_job)) == "duplicate_or_stale" + assert assert_ready(attachment, 2) == newer + assert len(state(attachment, 1)[1]) == 4 + results["duplicate_reorder_new_version"] = "pass" + + attachment, job = create_case() + with pool().connection() as conn: + claimed = claim_job(conn, job, 1) + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=5, + error_code="invalid_image", permanent=True, + ) == "failed" + attempt = replay_failed(conn, job, "lab5-test", "verified synthetic replay") + assert attempt == 2 + assert process_message(item(job, attempt)) == "ready" + assert_ready(attachment, 1) + with pool().connection() as conn: + count = conn.execute( + "SELECT count(*) AS n FROM attachment_job_replays WHERE job_id=%s", (job,) + ).fetchone()["n"] + assert count == 1 + results["failed_registry_operator_replay"] = "pass" + + attachment, job = create_case() + with pool().connection() as conn: + claimed = claim_job(conn, job, 1) + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=2, + delay_seconds=0.01, error_code="storage_unavailable", + ) == "queued" + time.sleep(0.03) + with pool().connection() as conn: + claimed = claim_job(conn, job, 2) + assert claimed is not None + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=2, + error_code="storage_unavailable", + ) == "failed" + attempt = replay_failed(conn, job, "lab5-test", "retry budget reset after repair") + assert attempt == 3 + assert process_message(item(job, attempt)) == "ready" + assert_ready(attachment, 1) + results["bounded_transient_retry_then_replay"] = "pass" + + print(json.dumps(results, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_publish_confirm_gap.py b/student-tests/test_publish_confirm_gap.py new file mode 100644 index 0000000..7524a49 --- /dev/null +++ b/student-tests/test_publish_confirm_gap.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Show that broker confirm without outbox confirmation safely republishes.""" + +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def compose(*args, input_data=None): + return subprocess.run( + ["docker", "compose", "-f", "compose.yaml", "-f", "compose.telemetry.yaml", *args], cwd=ROOT, + input=input_data, check=True, + ) + + +def main(): + args = arguments("Lab6 publish/confirm gap") + client = Client(args.base_url, args.ca_file) + compose("stop", "worker", "publisher") + attachment = None + try: + alice, bob, _eve, chat = client.bootstrap() + attachment, _original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not queued") + source = (ROOT / "student-tests/publish_gap_inner.py").read_bytes() + compose( + "run", "--rm", "--no-deps", "-T", + "-e", f"FAULT_ATTACHMENT_ID={attachment['id']}", "publisher", "python", "-", + input_data=source, + ) + finally: + compose("up", "-d", "publisher", "worker") + ready = client.wait_ready(bob, attachment) + check(ready["status"] == "ready", "republished job did not finish") + check(len(ready["variants"]) == 3, "duplicate delivery changed variant set") + print("PASS: confirmed broker copy republished after lost DB confirm; worker reached ready once") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_queue_incident.py b/student-tests/test_queue_incident.py new file mode 100644 index 0000000..d112841 --- /dev/null +++ b/student-tests/test_queue_incident.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Stop worker, build a real durable backlog, then verify queue-lag recovery.""" + +import json +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +sys.path.insert(0, str(ROOT / "scripts")) +from client import Client # noqa: E402 +from slo_watch import snapshot # noqa: E402 + + +FILES = ["-f", "compose.yaml"] +if (ROOT / "compose.local.yaml").exists(): + FILES += ["-f", "compose.local.yaml"] +FILES += ["-f", "compose.telemetry.yaml"] + + +def compose(*args): + subprocess.run(["docker", "compose", *FILES, "--env-file", ".env", *args], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + + +def now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def main(): + client = Client("http://127.0.0.1:8080") + alice, _bob, _eve, chat = client.bootstrap() + image = (ROOT / "tests/fixtures/sample.png").read_bytes() + samples = [] + stop = threading.Event() + + def watch(): + while not stop.is_set(): + samples.append(snapshot("http://127.0.0.1:9090")) + stop.wait(5) + + watcher = threading.Thread(target=watch, daemon=True) + watcher.start() + times = {"started": now()} + worker_stopped = False + try: + compose("stop", "worker") + worker_stopped = True + times["worker_stopped"] = now() + attachments = [] + for _ in range(8): + response = client.call( + "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=queue-probe.png", + 202, actor=alice, raw=image, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + attachments.append(response["json"]) + times["uploads_accepted"] = now() + time.sleep(35) + compose("up", "-d", "worker") + worker_stopped = False + times["worker_started"] = now() + for item in attachments: + client.wait_ready(alice, item, wait_seconds=120) + times["all_ready"] = now() + time.sleep(40) + finally: + if worker_stopped: + compose("up", "-d", "worker") + stop.set() + watcher.join(timeout=6) + states = [item["alerts"].get("Lab6QueueLag") for item in samples] + data = {"times": times, "queued_uploads": len(attachments), + "ready_uploads": len(attachments), "prometheus_samples": samples, + "queue_alert_states": sorted(set(states))} + path = ROOT / "evidence/queue-worker-stop-2026-10-02.json" + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"queued_uploads": len(attachments), "states": data["queue_alert_states"], + "times": times, "output": str(path)}, sort_keys=True)) + if "firing" not in states or states[-1] != "inactive": + raise SystemExit("queue lag alert did not fire and resolve") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_slo_alerts.py b/student-tests/test_slo_alerts.py new file mode 100644 index 0000000..2818276 --- /dev/null +++ b/student-tests/test_slo_alerts.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Reproduce a DB outage and capture Prometheus burn-rate transitions.""" + +import argparse +import json +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +COMPOSE_FILES = ["-f", "compose.yaml"] +if (ROOT / "compose.local.yaml").exists(): + COMPOSE_FILES += ["-f", "compose.local.yaml"] +COMPOSE_FILES += ["-f", "compose.telemetry.yaml"] +sys.path.insert(0, str(ROOT / "scripts")) +from slo_watch import snapshot # noqa: E402 + + +PROBE = r''' +import base64, json, sys, time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +end = time.monotonic() + int(sys.argv[1]) +events = [] +headers = {"Authorization": "Basic " + base64.b64encode(b"lab6_probe:synthetic-invalid-password").decode()} +def run_one(_): + result = [] + while time.monotonic() < end: + req = Request("http://127.0.0.1:8080/api/v1/auth/sessions", data=b"", headers=headers, method="POST") + try: + with urlopen(req, timeout=30) as response: + code = response.status + except HTTPError as exc: + code = exc.code + except OSError: + code = 0 + result.append([datetime.now(timezone.utc).isoformat(timespec="seconds"), code]) + time.sleep(0.4) + return result +with ThreadPoolExecutor(max_workers=1) as pool: + for batch in pool.map(run_one, range(1)): + events.extend(batch) +print(json.dumps({"events": sorted(events)})) +''' + + +def utc_now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def compose(*args): + subprocess.run( + ["docker", "compose", *COMPOSE_FILES, "--env-file", ".env", *args], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case", choices=("short", "long"), required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + pre, outage, post = (15, 50, 90) if args.case == "short" else (15, 180, 130) + duration = pre + outage + post + 10 + samples, watch_errors, times = [], [], {"started": utc_now()} + stop_watching = threading.Event() + + def watch(): + while not stop_watching.is_set(): + try: + samples.append(snapshot("http://127.0.0.1:9090")) + except (OSError, ValueError, KeyError) as exc: + watch_errors.append({"timestamp": utc_now(), "type": type(exc).__name__}) + stop_watching.wait(5) + + watcher = threading.Thread(target=watch, daemon=True) + watcher.start() + command = [ + "docker", "compose", *COMPOSE_FILES, "--env-file", ".env", "exec", "-T", + "api", "python", "-u", "-", str(duration), + ] + probe = subprocess.Popen(command, cwd=ROOT, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + probe.stdin.write(PROBE) + probe.stdin.close() + db_stopped = False + try: + time.sleep(pre) + times["stop_requested"] = utc_now() + compose("stop", "postgres") + db_stopped = True + times["db_stopped"] = utc_now() + time.sleep(outage) + times["restart_requested"] = utc_now() + compose("up", "-d", "--wait", "postgres") + db_stopped = False + times["db_healthy"] = utc_now() + time.sleep(post) + finally: + if db_stopped: + compose("up", "-d", "--wait", "postgres") + times["db_healthy_after_cleanup"] = utc_now() + stop_watching.set() + watcher.join(timeout=6) + + output = probe.stdout.read() + error = probe.stderr.read() + probe.wait(timeout=15) + result = json.loads(output) if probe.returncode == 0 else {"error": error[-500:]} + result.update({"case": args.case, "planned_seconds": {"pre": pre, "outage": outage, "post": post}, + "times": times, "prometheus_samples": samples, "watch_errors": watch_errors}) + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + states = {name: {sample["alerts"].get(name) for sample in samples} + for name in ("Lab6HighHttpErrorRate", "Lab6ApiBurnShort", "Lab6ApiBurnLong")} + codes = {event[1] for event in result.get("events", [])} + print(json.dumps({"case": args.case, "codes": sorted(codes), + "states": {name: sorted(values) for name, values in states.items()}, + "samples": len(samples), "output": str(path)}, sort_keys=True)) + expected = {"Lab6HighHttpErrorRate", "Lab6ApiBurnShort"} + if args.case == "long": + expected.add("Lab6ApiBurnLong") + if probe.returncode or not {401, 503} <= codes or any("firing" not in states[name] for name in expected): + raise SystemExit("incident did not demonstrate the expected HTTP SLO transitions") + if any(sample["alerts"].get(name) != "inactive" for name in expected for sample in samples[-3:]): + raise SystemExit("alerts did not resolve after database recovery") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_staging_cleanup.py b/student-tests/test_staging_cleanup.py new file mode 100644 index 0000000..4dddb28 --- /dev/null +++ b/student-tests/test_staging_cleanup.py @@ -0,0 +1,46 @@ +"""Run inside the cleanup image against an isolated pair of staging keys.""" + +import time +import sys +from io import BytesIO +from pathlib import Path +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.db import pool +from src.pipeline_jobs import reserve_upload +from src.staging_cleanup import BUCKET, clean_once, storage + + +client = storage() +orphan_id, active_id = uuid4(), uuid4() +orphan_key, active_key = f"staging/{orphan_id}", f"staging/{active_id}" + + +def exists(key): + return any(obj.object_name == key for obj in client.list_objects(BUCKET, prefix=key)) + + +try: + for key in (orphan_key, active_key): + client.put_object(BUCKET, key, BytesIO(b"test"), 4, content_type="image/png") + with pool().connection() as conn: + reserve_upload(conn, active_id, active_key, lease_seconds=60) + time.sleep(2) + clean_once(client, grace_seconds=1) + assert not exists(orphan_key), "old orphan was not removed" + assert exists(active_key), "active upload was removed" + + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_uploads SET lease_expires_at=clock_timestamp()-interval '1 second' WHERE id=%s", + (active_id,), + ) + clean_once(client, grace_seconds=1) + assert not exists(active_key), "expired orphan was not removed" + print("PASS: cleanup removes old orphans and preserves active uploads") +finally: + for key in (orphan_key, active_key): + client.remove_object(BUCKET, key) + with pool().connection() as conn: + conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (active_id,)) diff --git a/student-tests/test_telemetry.py b/student-tests/test_telemetry.py new file mode 100644 index 0000000..d5b8837 --- /dev/null +++ b/student-tests/test_telemetry.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""The durable job carrier preserves one trace across broker boundaries.""" + +import json +import sys +import unittest +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import Mock, patch +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from opentelemetry import trace # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from opentelemetry.trace import SpanKind # noqa: E402 +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator # noqa: E402 + + +exporter = InMemorySpanExporter() +provider = TracerProvider() +provider.add_span_processor(SimpleSpanProcessor(exporter)) +trace.set_tracer_provider(provider) + +from src.pipeline_jobs import job_payload # noqa: E402 +from src import worker # noqa: E402 + + +class TelemetryTest(unittest.TestCase): + def setUp(self): + exporter.clear() + + def test_failed_job_has_low_cardinality_metric_and_error_span(self): + item = { + "job_id": str(uuid4()), "attachment_id": str(uuid4()), + "type": "attachment.process.v1", "attempt": 1, + } + counter, duration = Mock(), Mock() + with patch.object(worker, "process_message", return_value="failed"), \ + patch.object(worker, "job_instruments", return_value=(counter, duration)): + self.assertEqual(worker.traced_process_message(item), "failed") + labels = {"job_type": "attachment.process.v1", "outcome": "failed"} + counter.add.assert_called_once_with(1, labels) + self.assertEqual(duration.record.call_args.args[1], labels) + self.assertEqual(exporter.get_finished_spans()[-1].status.status_code.name, "ERROR") + + def test_upload_publish_consume_and_retry_share_causal_trace(self): + propagator = TraceContextTextMapPropagator() + tracer = trace.get_tracer(__name__) + row = { + "job_id": uuid4(), "attachment_id": uuid4(), "pipeline_version": 1, + "attempt": 1, "created_at": datetime.now(timezone.utc), + } + + with tracer.start_as_current_span("upload") as upload: + upload_carrier = {} + propagator.inject(upload_carrier) + row.update(upload_carrier) + + producer_parent = propagator.extract(upload_carrier) + with tracer.start_as_current_span("publish", context=producer_parent, kind=SpanKind.PRODUCER) as producer: + producer_carrier = {} + propagator.inject(producer_carrier) + message = {**job_payload(row), **producer_carrier} + + with patch.object(worker, "process_message", return_value="queued"): + self.assertEqual(worker.traced_process_message(worker.parse_message(json.dumps(message).encode())), "queued") + # A managed retry uses the same durable upload parent but a new + # publisher attempt; both worker attempts remain in one trace. + row["attempt"] = 2 + retry_parent = propagator.extract(upload_carrier) + with tracer.start_as_current_span("retry_publish", context=retry_parent, + kind=SpanKind.PRODUCER) as retry_producer: + retry_carrier = {} + propagator.inject(retry_carrier) + retry_message = {**job_payload(row), **retry_carrier} + self.assertEqual(worker.traced_process_message(worker.parse_message(json.dumps(retry_message).encode())), "queued") + + spans = exporter.get_finished_spans() + consumers = [span for span in spans if span.name == "attachments.process"] + self.assertEqual(len(consumers), 2) + self.assertEqual(consumers[0].parent.span_id, producer.get_span_context().span_id) + self.assertEqual(consumers[1].parent.span_id, retry_producer.get_span_context().span_id) + self.assertEqual(producer.parent.span_id, upload.get_span_context().span_id) + self.assertEqual(retry_producer.parent.span_id, upload.get_span_context().span_id) + self.assertEqual({span.context.trace_id for span in spans}, {upload.get_span_context().trace_id}) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_upload_handoff.py b/student-tests/test_upload_handoff.py new file mode 100644 index 0000000..8a26182 --- /dev/null +++ b/student-tests/test_upload_handoff.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Exercise 202 handoff while RabbitMQ and worker are unavailable.""" + +import subprocess +import sys +import time +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + + +def compose(*args): + subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, + ) + + +def ready_after_restart(client): + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + time.sleep(1) + raise Failure("API did not recover after restart") + + +def main(): + args = arguments("Lab5 durable upload handoff with stopped broker and worker") + client = Client(args.base_url, args.ca_file) + compose("stop", "worker", "publisher", "broker") + try: + alice, bob, _eve, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", 409, actor=bob) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(message["attachment_ids"] == [attachment["id"]], "queued attachment cannot be linked") + + compose("up", "-d", "broker") + compose("restart", "api") + ready_after_restart(client) + check( + client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=bob, schema="Attachment")["json"]["status"] == "queued", + "accepted job disappeared before publisher restart", + ) + compose("up", "-d", "publisher", "worker") + client.wait_ready(bob, attachment) + result = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob) + check(result["bytes"] == original, "accepted original changed after recovery") + finally: + compose("up", "-d", "broker", "publisher", "worker") + print("PASS: accepted upload survived broker/API outage and reached ready") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_upload_queued.py b/student-tests/test_upload_queued.py new file mode 100644 index 0000000..ec6521d --- /dev/null +++ b/student-tests/test_upload_queued.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Public U2 check while publisher and worker are stopped.""" + +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def main(): + args = arguments("Lab5 durable queued upload") + client = Client(args.base_url, args.ca_file) + alice, bob, _eve, chat = client.bootstrap() + attachment, _content = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload did not enter queued state") + client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", 409, actor=bob) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(message["attachment_ids"] == [attachment["id"]], "queued attachment cannot be linked") + print("PASS: 202 queued upload is readable and linkable; content is pending") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_variants_api.py b/student-tests/test_variants_api.py new file mode 100644 index 0000000..fd4aee8 --- /dev/null +++ b/student-tests/test_variants_api.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Check every public variant against committed metadata and original bytes.""" + +import hashlib +import sys +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def main(): + args = arguments("Lab5 ready variants") + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + ready = client.wait_ready(bob, attachment) + variants = {item["name"]: item for item in ready["variants"]} + check(set(variants) == {"thumbnail", "crop", "optimized"}, "incomplete variant set") + expected = { + "thumbnail": ("image/jpeg", 256, 160), + "crop": ("image/jpeg", 128, 128), + "optimized": ("image/webp", 320, 200), + } + for name, (content_type, width, height) in expected.items(): + variant = variants[name] + check((variant["content_type"], variant["width"], variant["height"]) == + (content_type, width, height), f"wrong {name} metadata") + response = client.call( + "GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", actor=bob + ) + check(response["headers"].get_content_type() == content_type, f"wrong {name} MIME") + check(len(response["bytes"]) == variant["size_bytes"] and + hashlib.sha256(response["bytes"]).hexdigest() == variant["sha256"], + f"wrong {name} bytes") + client.call( + "GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", + 404, actor=eve, + ) + downloaded = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob) + check(downloaded["bytes"] == original, "original bytes changed") + print("PASS: all three variants match metadata; original preserved; outsider denied") + + +if __name__ == "__main__": + run(main)