From 7fe48e38c19a0aa346d25624e6cdff4ff3800b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Fri, 6 Nov 2026 11:12:38 +0300 Subject: [PATCH] refactor code --- REPORT.md | 71 +- compose.yaml | 1 + docs/observability.md | 2 +- docs/review.md | 36 + docs/slo-runbook.md | 10 +- .../pre-review/ready-scenario-failed.json | 300 +++++ evidence/pre-review/slo-analysis.json | 59 + evidence/pre-review/slo-incidents.json | 1170 +++++++++++++++++ evidence/pre-review/slo-ready-incident.json | 211 +++ evidence/review-checks.json | 109 ++ evidence/review-dashboard.json | 137 ++ evidence/review-db-trace.json | 909 +++++++++++++ evidence/review-observability.md | 29 + evidence/review-resilience.json | 27 + evidence/review-retry-job.json | 9 + evidence/review-retry-summary.json | 27 + evidence/review-retry-trace.json | 870 ++++++++++++ evidence/review-tempo-load.json | 17 + evidence/review-tempo-outage.json | 245 ++++ evidence/review-tempo-persistence.json | 93 ++ evidence/slo-analysis.json | 44 +- evidence/slo-incidents.json | 974 +++++++++----- evidence/slo-ready-incident.json | 283 ++-- evidence/slo-track.md | 20 +- infra/prometheus/alerts.yaml | 10 + migrations/009_slo_commits.sql | 11 + pyproject.toml | 6 +- src/broker.py | 6 +- src/main.py | 11 +- src/outbox.py | 21 +- src/storage/postgres.py | 37 + src/telemetry.py | 31 +- src/worker.py | 36 +- student-tests/capture_observability.py | 41 + student-tests/check_dashboard.py | 18 + student-tests/check_retry_trace.py | 31 + student-tests/pipeline_critical.py | 12 +- student-tests/retry_evidence.py | 30 + student-tests/review_checks.py | 22 + student-tests/review_queries.py | 13 + student-tests/review_resilience.py | 20 + student-tests/slo_analyze.py | 8 + student-tests/slo_commit_integration.py | 45 + student-tests/slo_incidents.py | 20 +- student-tests/slo_ready_incident.py | 16 +- student-tests/telemetry_alerts.test.yaml | 31 + student-tests/telemetry_unit.py | 38 + student-tests/tempo_outage.py | 60 + student-tests/trace_retry.py | 7 + uv.lock | 4 +- 50 files changed, 5674 insertions(+), 564 deletions(-) create mode 100644 docs/review.md create mode 100644 evidence/pre-review/ready-scenario-failed.json create mode 100644 evidence/pre-review/slo-analysis.json create mode 100644 evidence/pre-review/slo-incidents.json create mode 100644 evidence/pre-review/slo-ready-incident.json create mode 100644 evidence/review-checks.json create mode 100644 evidence/review-dashboard.json create mode 100644 evidence/review-db-trace.json create mode 100644 evidence/review-observability.md create mode 100644 evidence/review-resilience.json create mode 100644 evidence/review-retry-job.json create mode 100644 evidence/review-retry-summary.json create mode 100644 evidence/review-retry-trace.json create mode 100644 evidence/review-tempo-load.json create mode 100644 evidence/review-tempo-outage.json create mode 100644 evidence/review-tempo-persistence.json create mode 100644 migrations/009_slo_commits.sql create mode 100644 student-tests/capture_observability.py create mode 100644 student-tests/check_dashboard.py create mode 100644 student-tests/check_retry_trace.py create mode 100644 student-tests/retry_evidence.py create mode 100644 student-tests/review_checks.py create mode 100644 student-tests/review_queries.py create mode 100644 student-tests/review_resilience.py create mode 100644 student-tests/slo_commit_integration.py create mode 100644 student-tests/telemetry_alerts.test.yaml create mode 100644 student-tests/tempo_outage.py diff --git a/REPORT.md b/REPORT.md index 6a79e7e..6adc761 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,57 +1,66 @@ # Отчёт — лабораторная 6 -- Автор, группа: Kozyrev, BVT2301 -- Целевая оценка: 5, трек 2 — SLO и обнаружение инцидентов -- Commit/PR: TODO после публикации -- Источники: [контракт телеметрии](contracts/observability.md), [OTel Collector](https://opentelemetry.io/docs/collector/configuration/), [OTel Python exporters](https://opentelemetry.io/docs/languages/python/exporters/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), исходники lab5 commit `6bc0712`. При реализации использован Codex. +- Автор: Kozyrev, BVT2301. +- Цель: уровень 5, трек 2 — SLO и обнаружение инцидентов. +- Рабочая ветка: `solution/BVT2301-Kozyrev`. Результаты повторной проверки — [docs/review.md](docs/review.md). Независимое прохождение runbook другим студентом не подтверждено; полностью закрытым уровень 5 не объявляется. +- При реализации и проверке использован Codex. Исходная основа: lab5, commit `6bc0712`. -## Запуск +## Воспроизводимый запуск -Нужны Python 3, Docker Engine и Compose v2. API слушает `127.0.0.1:8080`, Grafana — `127.0.0.1:3000`. Версии образов закреплены в [Dockerfile](Dockerfile) и [compose.yaml](compose.yaml). +Нужны Docker Engine с Compose v2 и Python 3 для публичных тестов. Для локальных SDK-тестов нужен Python ≥3.12 согласно [pyproject.toml](pyproject.toml); их также можно запускать внутри контейнера. ```sh python3 scripts/init-secrets.py docker compose config --quiet docker compose up -d --build -docker compose ps -a -curl --fail http://localhost:8080/health/ready +# Дождаться успешного ответа; старт API может занять время: +python3 - <<'PY' +import time, urllib.request +for attempt in range(90): + try: + urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2).close() + break + except OSError: + time.sleep(1) +else: + raise SystemExit('API readiness timeout; inspect docker compose ps -a') +PY make check make test ``` -Миграции и создание S3 bucket выполняют сервисы `migrate` и `s3-init`. Секреты создаются в игнорируемом `secrets/`; `.env` для значений Compose по умолчанию не обязателен. В Grafana логин `admin`, пароль — в локальном `secrets/grafana_password`. `docker compose down` сохраняет volumes; `docker compose down -v` намеренно удаляет все именованные volumes этого проекта, включая учебные данные и историю телеметрии. +Миграции выполняет `migrate`, bucket создаёт `s3-init`; [Compose](compose.yaml) закрепляет образы и зависимости. API: `127.0.0.1:8080`; Grafana: `127.0.0.1:3000`, логин `admin`, пароль в игнорируемом `secrets/grafana_password`. `.env` при значениях по умолчанию не обязателен. `docker compose down` сохраняет volumes; `down -v` удаляет данные и историю. -## Архитектура и решения +Для полной проверки OpenAPI установите `requirements-dev.txt` в `.venv`, как описано в [README](README.md), затем `make validate PYTHON=.venv/bin/python`. Короткие проверки повторяются командой `python3 student-tests/review_checks.py`; журнал сохраняется в [review-checks.json](evidence/review-checks.json). Инциденты запускаются отдельно и намеренно ухудшают учебный стенд. -Клиент обращается через HAProxy к двум API-репликам. PostgreSQL хранит бизнес-данные, задания и outbox; SeaweedFS — байты вложений. Dispatcher публикует события из outbox в RabbitMQ, worker формирует варианты; reconcile выполняет сверку. S3 и PostgreSQL не образуют общую транзакцию. Из lab5 перенесены исходники, миграции, инфраструктура и собственные тесты; контракты и публичные тесты lab6 сохранены. +## Уровень 3 -API, outbox и worker отправляют OTLP через Collector в Prometheus, Loki и Tempo; Grafana использует эти хранилища через provisioned datasources. [Dashboard](infra/grafana/dashboards/messenger.json), [правила alerts](infra/prometheus/alerts.yaml), конфигурации и миграция context находятся в репозитории. Схема сигналов, запросы, SLI/SLO, ограничения и сценарии — в [docs/observability.md](docs/observability.md). `.env.example` описывает логические настройки; Compose часть значений задаёт явно. OTLP включён в Compose для API, worker и outbox. +HAProxy распределяет запросы между двумя API. PostgreSQL хранит бизнес-данные, outbox и задания; SeaweedFS — файлы; RabbitMQ доставляет задания worker. Сигналы API/worker/outbox идут по OTLP в Collector. Prometheus хранит метрики, Loki — логи, Tempo — трассы; Grafana запрашивает эти хранилища. Источники и dashboard загружаются provisioning; нужные данные находятся в named volumes. Исходники: [compose.yaml](compose.yaml), [Collector](infra/otel/collector.yaml), [dashboard](infra/grafana/dashboards/messenger.json). -## Проверки и доказательства +[Telemetry](src/telemetry.py) ограничивает metric labels, очищает log record перед stdout и OTLP. Неизвестные HTTP methods сводятся к `_OTHER`. Jobs имеют `success`, `failed`, `skipped`: повторная доставка без нового результата не считается успешной обработкой. Проверки реального SDK exporter, W3C propagation и исходов job — [telemetry_unit.py](student-tests/telemetry_unit.py), [pipeline_critical.py](student-tests/pipeline_critical.py). -| Критерий | Команда/сценарий | Результат | -| --- | --- | --- | -| Целостность и smoke | `make check`, `make validate PYTHON=.venv/bin/python`, `make test` | PASS; smoke повторён после восстановления. | -| Конфигурации | `docker compose config --quiet`; штатные проверки Collector, Loki, Tempo и `promtool` | PASS; Grafana API вернул три datasource и dashboard, запросы панелей выполнены. | -| Контекст и конфиденциальность | `.venv/bin/python student-tests/telemetry_unit.py` | 3 теста PASS: parent/child через AMQP, ограниченные поля логов и шаблон маршрута метрики. | -| Обработка вложений | `.venv/bin/python student-tests/pipeline_critical.py` | 8 тестов PASS. | -| Данные и реплики | `make test-persistence ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | Оба PASS; две реплики восстановлены. Фактический состав hook описан в скриптах; отдельного снимка состояний до/после для persistence нет. | -| Связь API → worker, включая retry | upload с тестовой инъекцией crash worker | ready на второй попытке; 2 producer и 2 consumer spans в одном trace. | -| Alert очереди и alert ошибок API | остановка worker; блокировка таблицы users при нагрузке; `promtool test rules student-tests/alerts.test.yaml` в контейнере | Оба наблюдались в firing и отсутствовали после восстановления; синтетический тест финальных правил PASS. | -| Отказ хранилища telemetry | остановить Tempo, выполнить smoke и отдельную нагрузку 75 с | smoke PASS; очередь ограничена, зафиксировано переполнение и потери traces. | +Повторные публичные проверки сохранности и отказа реплики: [review-resilience.json](evidence/review-resilience.json), сценарий [review_resilience.py](student-tests/review_resilience.py). -Идентификаторы трасс, результаты, формулы и границы проверки приведены в [evidence/observability.md](evidence/observability.md). Raw summaries нагрузки: [инцидент БД](evidence/db-incident-load-repeat.json), [отказ Tempo](evidence/tempo-outage-load.json). Полный набор наблюдений живёт в локальных volumes, а не в Git. +## Уровень 4 -## Трек 5: SLO и обнаружение инцидентов +Контекст сохраняется вместе с заданием и outbox, передаётся через AMQP headers и восстанавливается worker. Ошибочный producer span получает ERROR без текста исключения. Dashboard содержит histogram p50/p95/p99, ошибки, очередь, pool и worker saturation; все запросы проверяет [check_dashboard.py](student-tests/check_dashboard.py), результаты — [review-dashboard.json](evidence/review-dashboard.json). Описание метрик, единиц, запросов и ограничений — [observability.md](docs/observability.md). -Заданы учебные SLO 99% за 24 часа для API и времени готовности принятых вложений. Для каждого настроены быстрые и медленные multiwindow burn-rate alerts; формулы, допущения и пошаговая диагностика — в [runbook](docs/slo-runbook.md). Последние 30 секунд заданий исключены из зрелой когорты; просроченные, failed и неготовые задания входят в ошибки SLO. Моменты начала и окончания приблизительны из-за транзакционных границ. При недоступном outbox счётчики устаревают. +Время ready теперь вычисляется между commit принятия и commit первого ready в PostgreSQL. [Миграция 009](migrations/009_slo_commits.sql) сохраняет отдельную когорту; outbox материализует timestamps и однократно фиксирует результат 30-секундного срока. `track_commit_timestamp=on` включён в Compose. Старые задания не backfill-ятся приближёнными датами. [Интеграционный тест](student-tests/slo_commit_integration.py) проверяет commit-границы и неизменность good/bad без изменения бизнес-данных. -Повторяемые сценарии и наблюдения — в [отчёте трека](evidence/slo-track.md), исходные отсчёты — [API](evidence/slo-incidents.json), [расчёт](evidence/slo-analysis.json) и [вложения](evidence/slo-ready-incident.json). При короткой и длительной блокировке API fast alert обнаружен через 55,088 и 50,278 секунды; slow в первом случае впервые появился после конца инцидента (одна новая тревога в фазе восстановления), во втором — через 90,428 секунды. На коротком baseline из 13 проб не наблюдалось burn alerts; долгосрочная частота ложных срабатываний не измерена. При тесте восьми вложений обнаружение — 40,279 секунды после последнего принятия, после запуска worker все восемь стали ready. Опросный метод, ограничения и заметный скачок UTC-часов описаны в отчёте трека. Проведена собственная проверка runbook по Loki/Tempo; прохождение другим студентом не подтверждено. +Повторный crash/retry проверен [сценарием](student-tests/retry_evidence.py): [job/attempt/restart](evidence/review-retry-job.json), [логи и trace](evidence/review-retry-trace.json), [проверка дерева spans](evidence/review-retry-summary.json). Незавершённые spans аварийной попытки могут потеряться при `os._exit`; сохранённые consumer spans не доказывают полноту упавшей попытки. -Проверка правил: `docker run --rm -v "$PWD:/workspace:ro" --entrypoint promtool prom/prometheus:v3.2.1 test rules /workspace/student-tests/burn_rules.test.yaml`. Синтетические проверки нормальной нагрузки, всплеска, устойчивой деградации и малого числа событий прошли. +Сценарии очереди и БД сохранены в [slo-ready-incident.json](evidence/slo-ready-incident.json) и [slo-incidents.json](evidence/slo-incidents.json); коррелированный лог 503 и trace — [review-db-trace.json](evidence/review-db-trace.json). Ограниченный отказ Tempo воспроизводится [tempo_outage.py](student-tests/tempo_outage.py), включая baseline счётчиков, их дельты и память Collector: [сырые результаты](evidence/review-tempo-outage.json), [разбор](evidence/review-observability.md). Старые эксперименты и их ограничения сохранены отдельно в [observability.md](evidence/observability.md). -## Эксперименты и ограничения +## Уровень 5, трек 2 -Docker Engine 29.8.0, `aarch64`; Docker VM сообщил 10 CPU и 10 416 644 096 байт RAM. Лимиты контейнеров указаны в Compose: например, Collector — 256 MiB, каждая API-реплика — 0,5 CPU и 512 MB. Ограниченная нагрузка была closed-loop; прогрев и статистические повторы для сравнения производительности не проводились. Числа не являются оценкой максимальной пропускной способности. В отчёте о первом опыте блокировки БД alert не успел перейти в firing; после изменения выдержки до 10 секунд повторный опыт показал firing и восстановление. +Гипотеза: multiwindow burn-rate alerts обнаруживают контролируемый расход бюджета и снимаются после восстановления короткого окна. Учебные цели — 99% API без 5xx и 99% принятых файлов ready ≤30 с, окно SLO 24 часа. Порог 30 с выбран относительно 60-секундного ожидания smoke; 99% — учебная цель, не исследование пользователей. Формулы, числители, знаменатели, low-volume guard и диагностика приведены в [runbook](docs/slo-runbook.md). -`messenger_job_ready_duration_seconds` измеряет время от создания DB-записи о вложении до фиксации ready; оно включает время до commit принятия. Репрезентативность 24-часового окна и долгосрочная частота ложных срабатываний не проверены. Данные queue age обновляет outbox: при его остановке gauge устаревает. RSS/CPU worker не включают отдельный render-процесс; pool waiting зависит от внутреннего API закреплённой версии asyncpg. Один Docker-host и единственные экземпляры DB, broker и LB не обеспечивают HA при их потере. Три замера RAM при отказе Tempo не доказывают поведение для произвольной длительности; переполнение очереди приводит к потере spans. При необходимости переносимой защиты следует отдельно сохранить наблюдения и подтвердить состояние alerts и трасс после нового clone. +Методика, свежие результаты, оценка задержанных/ложных тревог и ограничения — [slo-track.md](evidence/slo-track.md). Расчёт выполняет [slo_analyze.py](student-tests/slo_analyze.py). Старый опыт имел расхождение приращений UTC/monotonic до 83,075 с: данные сохранены в [pre-review](evidence/pre-review/slo-analysis.json), но не принимаются как надёжный замер длительностей. Причину расхождения установить нельзя. + +Снятие alert из-за недостатка новых событий не означает восстановления суточного SLO. Короткий baseline не измеряет долгосрочную частоту ложных срабатываний. Подготовлен протокол независимой проверки runbook; фактического прохождения другим студентом пока нет. + +## Практические ограничения и источники + +Один Docker-host и единственные DB/broker/LB не обеспечивают HA при их потере. CPU/RSS worker не включают дочерний render; pool waiting использует внутреннюю очередь закреплённой версии asyncpg. Потеря отдельных histogram samples возможна при crash после commit; durable SLO-когорта сохраняется. При разрыве экспорта `increase` может пропускать события; stale/unknown alerts показывают неполноту наблюдения. Commit timestamps PostgreSQL хранятся ограниченное время, поэтому outbox сохраняет их в таблицу. Репрезентативные 24 часа и долговременная устойчивость по коротким экспериментам не доказаны. + +Источники: [контракт](contracts/observability.md), [PostgreSQL commit timestamps](https://www.postgresql.org/docs/17/functions-info.html#FUNCTIONS-COMMIT-TIMESTAMP), [OTel Collector](https://opentelemetry.io/docs/collector/configuration/), [OTel Python exporters](https://opentelemetry.io/docs/languages/python/exporters/), [W3C Trace Context](https://www.w3.org/TR/trace-context/), [Prometheus histograms](https://prometheus.io/docs/practices/histograms/), [SRE Workbook: SLO alerts](https://sre.google/workbook/alerting-on-slos/). diff --git a/compose.yaml b/compose.yaml index 551c0cd..4d58c8e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -114,6 +114,7 @@ services: postgres: image: postgres:17.6-alpine + command: ["postgres", "-c", "track_commit_timestamp=on"] environment: POSTGRES_DB: messenger POSTGRES_USER: messenger diff --git a/docs/observability.md b/docs/observability.md index 60a2da1..9318302 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -8,7 +8,7 @@ API, worker и outbox отправляют OTLP/HTTP в Collector. Prometheus х При upload текущий W3C trace context сохраняется в `attachment_jobs` одной транзакцией с outbox. Dispatcher извлекает его и публикует сообщение с дочерним producer span и AMQP headers; worker извлекает контекст и создаёт consumer span. Повторная публикация и retry используют сохранённый контекст исходной загрузки. Каждая попытка имеет собственные spans; они являются потомками исходной транзакции загрузки. Baggage не переносится. Контекст — технические данные, не содержимое изображения. -Dashboard использует счётчики и гистограммы с единицами секунд: p50/p95/p99 HTTP вычисляются через `histogram_quantile` по сумме buckets, не по среднему перцентилей. `messenger_job_ready_duration_seconds` измеряет от времени создания записи о вложении в DB до фиксации ready; эта оценка включает время до commit принятия и не претендует на точную границу durable acceptance. Queue depth и age — число и возраст самого старого задания в состоянии queued из PostgreSQL, включая ожидающие retry; это не длина очереди RabbitMQ. Данные обновляет outbox. DB pool отражает пул каждой API-реплики; число ожидающих читается из внутренней очереди asyncpg 0.31.0 и зависит от этой версии. Worker busy/capacity — число занятых executor threads и их лимит; повторная доставка уже завершённой задачи учитывается как успешная обработка. CPU и RSS — процессы API/worker/outbox, без дочерних процессов render. При остановленном outbox queue gauge устаревает. +Dashboard использует счётчики и гистограммы с единицами секунд: p50/p95/p99 HTTP вычисляются через `histogram_quantile` по сумме buckets, не по среднему перцентилей. `messenger_job_ready_duration_seconds` измеряет разность commit публикации ready и commit принятия задания из `attachment_slo` (миграция 009). Оба момента берутся из PostgreSQL; часы worker не участвуют. Гистограмма записывается после успешного commit: аварийное завершение worker между commit и записью метрики может потерять этот histogram sample; DB-когорта SLO при этом сохраняется. Queue depth и age — число и возраст самого старого задания в состоянии queued из PostgreSQL, включая ожидающие retry; это не длина очереди RabbitMQ. Данные обновляет outbox. DB pool отражает пул каждой API-реплики; число ожидающих читается из внутренней очереди asyncpg 0.31.0 и зависит от этой версии. Worker busy/capacity — число занятых executor threads и их лимит; повторная доставка без нового результата имеет outcome `skipped`, успешный commit — `success`, ошибка — `failed`. CPU и RSS — процессы API/worker/outbox, без дочерних процессов render. Queue-метрики имеют label `queue="attachment.process.v1"`. `messenger_queue_sample_time_seconds` показывает время последнего успешного обновления; `MessengerQueueTelemetryStale` сигнализирует об устаревании/отсутствии ряда. SLI: доля ответов 5xx среди всех HTTP бизнес-запросов API, исключая health. Целевой SLO: не менее 99% запросов без 5xx за скользящие 24 часа; ошибки балансировщика, которые не дошли до API, SLI не учитывает. Целевое значение выбрано для учебного стенда, а не по измеренным требованиям пользователей. Второй SLO и расчёт burn-rate описаны в [runbook](slo-runbook.md); его DB-счётчики учитывают и невыполненные в срок задания. Alert `MessengerHighErrorRate` срабатывает при >5% за минуту в течение 10 секунд; это оперативный порог, не burn-rate alert. `MessengerQueueLag` срабатывает при возрасте queued >30 секунд в течение 15 секунд. Состояния доступны в Prometheus через Grafana Explore или локальный API Prometheus внутри контейнера. diff --git a/docs/review.md b/docs/review.md new file mode 100644 index 0000000..20489b9 --- /dev/null +++ b/docs/review.md @@ -0,0 +1,36 @@ +# Проверка по критериям README + +Проверялись [README](../README.md), [контракт наблюдаемости](../contracts/observability.md), исходный [REPORT](../REPORT.md), код и локальный Docker-стенд. Автоматический smoke не выставляет оценку за наблюдаемость. + +## Итог по уровням + +- **3: подтверждён после исправлений.** [Короткие проверки](../evidence/review-checks.json), [публичные persistence/failover](../evidence/review-resilience.json), [коррелированные логи API/worker и trace](../evidence/review-retry-trace.json). +- **4: подтверждён в проверенном локальном стенде.** [Разбор инцидентов и отказа Tempo](../evidence/review-observability.md), [QueueLag/ready](../evidence/slo-ready-incident.json), [проверка retry-дерева](../evidence/review-retry-summary.json), [сохранность trace после рестарта Tempo](../evidence/review-tempo-persistence.json). +- **5: техническая часть трека 2 проверена, полный зачёт не подтверждён.** [Повторные измерения](../evidence/slo-track.md) и runbook есть; независимый диагноз другого студента не задокументирован. Итоговая подтверждённая оценка этой проверки — **4**. + +## Исправления + +| Уровень | Найденная ошибка | Исправление и проверка | +| --- | --- | --- | +| 3 | Allowlist применялся только к stdout; OTLP LoggingHandler получал произвольные extra и exception | Фильтр до обоих handlers; тест реального SDK log exporter с секретом в extra и exception — [telemetry_unit.py](../student-tests/telemetry_unit.py) | +| 3 | Произвольные HTTP method могли создавать новые metric labels | Стандартные методы и `_OTHER`; шаблон маршрута сохранён — [main.py](../src/main.py), [telemetry.py](../src/telemetry.py) | +| 3 | Повторная доставка без нового результата считалась успешной job | Отдельный исход `skipped`; `success` только после commit ready — [worker.py](../src/worker.py), [pipeline_critical.py](../student-tests/pipeline_critical.py) | +| 4 | Исключение publish оставляло producer span без ERROR | ERROR без экспорта текста исключения; regression test — [broker.py](../src/broker.py), [telemetry_unit.py](../student-tests/telemetry_unit.py) | +| 4 | Histogram ready начинался до durable acceptance и использовал часы worker | Оба конца — commit timestamps PostgreSQL; [миграция 009](../migrations/009_slo_commits.sql), [изолированный интеграционный тест](../student-tests/slo_commit_integration.py) | +| 4 | Queue gauge не имел queue name и мог незаметно устареть | Label queue, timestamp последнего обновления и stale/absent alert; [telemetry.py](../src/telemetry.py), [правила](../infra/prometheus/alerts.yaml) | +| 4 | Потери Tempo приписывались накопленным счётчикам без baseline | Повторяемый опыт с baseline, дельтами, ID Collector и замерами памяти — [tempo_outage.py](../student-tests/tempo_outage.py) | +| 5 | Ready SLO зависел от изменяемых status/updated_at | Отдельная зрелая когорта с однократно зафиксированным bad; retry/replay не меняют результат — [postgres.py](../src/storage/postgres.py) | +| 5 | Старые временные результаты использовались при расхождении UTC/monotonic | Старые отсчёты сохранены в [pre-review](../evidence/pre-review/slo-analysis.json); анализатор выявил расхождение 83,075 с и пометил их непригодными для elapsed-time выводов; причина не установлена | +| 5 | При исключении сценарий мог зависнуть, ожидая бесконечные клиентские потоки | STOP устанавливается до выхода из ThreadPoolExecutor, включая исключения — [slo_incidents.py](../student-tests/slo_incidents.py) | + +Дополнительный прогон выявил дефект теста очереди: требовалось одновременное firing Fast и QueueLag, хотя их окна различаются. Кроме того, выбирался первый metric series старого outbox после пересоздания. [Неудачный прогон](../evidence/pre-review/ready-scenario-failed.json) сохранён; сценарий теперь проверяет наблюдение обоих сигналов и снятие каждого, а абсолютные DB-счётчики читает через max по экземплярам (они представляют одну БД). + +## Ограничения, которые нельзя скрыть оценкой + +- Прохождение runbook другим студентом не подтверждено. Подготовлен [протокол независимой проверки](slo-runbook.md#проверка-другим-студентом); собственная проверка не заменяет внешнюю. +- 99%/24 часа — учебная цель; короткий опыт не доказывает выполнение суточного SLO. Baseline оценивает наблюдавшиеся ложные тревоги только за этот интервал. +- Старые задания до миграции 009 не имеют восстановимых commit timestamps и исключены из новой когорты. Исторические Prometheus-ряды до миграции нельзя смешивать с новой методикой для доказательства SLO. +- PostgreSQL не хранит commit timestamps бессрочно. Outbox сохраняет их в таблицу; если потерял источник до материализации, unknown alert показывает неполноту данных. При перезапуске экспортёра `increase` может не учесть события в разрыве наблюдения. +- Histogram sample может потеряться при crash после commit ready; durable DB-когорта сохраняется. Collector с ограниченной очередью также может терять spans. Это не «нулевые потери». + +Источник границ commit timestamps: [PostgreSQL 17](https://www.postgresql.org/docs/17/functions-info.html#FUNCTIONS-COMMIT-TIMESTAMP). Источник критериев оценки — локальные README и contracts, ссылки выше. diff --git a/docs/slo-runbook.md b/docs/slo-runbook.md index d449ed1..f6e2c07 100644 --- a/docs/slo-runbook.md +++ b/docs/slo-runbook.md @@ -4,7 +4,7 @@ ## Что означает сигнал -Два учебных SLO на скользящие 24 часа: 99% бизнес-запросов API без 5xx и 99% принятых вложений становятся ready не позднее чем через 30 секунд после создания записи о задании. API исключает health и ошибки LB до попадания в приложение; 4xx остаются в знаменателе. Для готовности учтены только задания, достигшие 30-секундного возраста: неготовые, failed и завершённые позднее срока считаются плохими. Счётчики вычисляет outbox по PostgreSQL, поэтому при его недоступности данные не обновляются; события между последним scrape старого процесса и первым scrape нового могут не попасть в расчёт `increase`. Момент создания записи немного предшествует commit принятия, а время ready фиксируется внутри транзакции до её commit: это приближение к пользовательским границам. Порог 30 секунд — половина 60-секундного бюджета smoke-теста для одного файла при свободном worker; это учебная цель, не измеренное требование пользователей. Цель 99% также выбрана для демонстрации, а не выведена из пользовательского исследования. +Два учебных SLO на скользящие 24 часа: 99% бизнес-запросов API без 5xx и 99% принятых вложений становятся ready не позднее чем через 30 секунд после commit принятия задания. API исключает health и ошибки LB до попадания в приложение; 4xx остаются в знаменателе. Для готовности учтены только задания, достигшие 30-секундного возраста: неготовые, failed и завершённые позднее срока считаются плохими. Счётчики вычисляет outbox по PostgreSQL, поэтому при его недоступности данные не обновляются; события между последним scrape старого процесса и первым scrape нового могут не попасть в расчёт `increase`. Миграция 009 добавляет `attachment_slo`: моменты acceptance и первого ready берутся из commit timestamps PostgreSQL и сохраняются outbox. Результат зрелой когорты фиксируется один раз; retry/replay не меняют историю. Старые задания до миграции исключены, поскольку восстановить точные времена их приёма нельзя. `track_commit_timestamp=on` задан в Compose. PostgreSQL удаляет старые commit timestamps при vacuum; outbox материализует их раз в цикл. Если не успел, растёт `messenger_slo_unknown`, срабатывает `MessengerReadyTimestampUnknown`: неизвестные события нельзя считать доказательством выполнения SLO. Источник: [PostgreSQL 17, commit timestamps](https://www.postgresql.org/docs/17/functions-info.html#FUNCTIONS-COMMIT-TIMESTAMP). Порог 30 секунд — половина 60-секундного бюджета smoke-теста для одного файла при свободном worker; это учебная цель, не измеренное требование пользователей. Цель 99% также выбрана для демонстрации, а не выведена из пользовательского исследования. Error budget каждого SLO — `1 − 0,99 = 0,01`. Burn rate — доля плохих событий за окно / 0,01. Приближение расхода бюджета `burn × окно / 24 часа` предполагает стабильную интенсивность событий; при сильно меняющейся нагрузке бюджет, выраженный в доле событий за 24 часа, расходуется иначе. Значение 1 означает, что при сохранении текущей доли плохих событий все допустимые ошибки будут израсходованы за период SLO. @@ -22,13 +22,13 @@ Error budget каждого SLO — `1 − 0,99 = 0,01`. Burn rate — доля ## Ready burn / очередь -1. Проверьте `sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m]))`, p95 ready и исходы jobs. Счётчик `messenger_slo_ready_bad_total` включает задания, нарушившие срок. Отсутствие событий не делает SLI хорошим: проверьте `messenger_queue_depth{service_name="messenger-outbox"}` и `messenger_queue_age_seconds{service_name="messenger-outbox"}`. +1. Сначала убедитесь, что `MessengerQueueTelemetryStale` и `MessengerReadyTimestampUnknown` не firing. Проверьте `sum(increase(messenger_slo_ready_eligible_total{service_name="messenger-outbox"}[1m]))`, p95 ready и исходы jobs. Счётчик `messenger_slo_ready_bad_total` включает задания, нарушившие срок. Отсутствие событий не делает SLI хорошим: проверьте `messenger_queue_depth{service_name="messenger-outbox"}` и `messenger_queue_age_seconds{service_name="messenger-outbox"}`. 2. В Loki откройте `{service_name="messenger-worker"}`, найдите событие и trace ID. В Tempo проверьте цепочку upload → producer → consumer → S3/DB; посмотрите busy/capacity и состояние broker/worker через `docker compose ps`. 3. В учебном опыте запустите остановленный worker, наблюдайте уменьшение глубины/возраста и новые ready. Проверьте resolved и метаданные вложений. Не удаляйте volumes и не запускайте replay вслепую. ## Повторение экспериментов -`python3 student-tests/slo_incidents.py` создаёт синтетические данные, ведёт три клиента и выполняет краткую и длительную блокировку таблицы `users`. Скрипт пишет временные отметки и наблюдения в [evidence/slo-incidents.json](../evidence/slo-incidents.json), не сохраняет tokens; опрашивает alert примерно каждые 5 секунд. `python3 student-tests/slo_analyze.py` строит сводку, включая срабатывания уже после завершения воздействия. Тест намеренно ухудшает локальный API и занимает несколько минут. Первая отметка воздействия — момент, когда скрипт увидел granted `AccessExclusiveLock`; окончание — возврат команды транзакции, поэтому обе оценки имеют задержку наблюдения. Длительности считаются по монотонным часам: при выполненном опыте системное UTC скачкообразно изменилось. Аварийное прерывание клиентской команды может оставить server-side `pg_sleep` до окончания заданного ограниченного времени; проверьте состояние блокировки. +`python3 student-tests/slo_incidents.py` создаёт синтетические данные, ведёт три клиента и выполняет краткую и длительную блокировку таблицы `users`. Скрипт пишет временные отметки и наблюдения в [evidence/slo-incidents.json](../evidence/slo-incidents.json), не сохраняет tokens; опрашивает alert примерно каждые 5 секунд. `python3 student-tests/slo_analyze.py` строит сводку, включая срабатывания уже после завершения воздействия. Тест намеренно ухудшает локальный API и занимает несколько минут. Первая отметка воздействия — момент, когда скрипт увидел granted `AccessExclusiveLock`; окончание — возврат команды транзакции, поэтому обе оценки имеют задержку наблюдения. Анализатор сравнивает приращения UTC и монотонных часов и сохраняет максимальный фактический интервал опроса. Если `timing_quality.valid_for_elapsed_time_claims=false`, опыт повторяют; причину расхождения часов по этим данным установить нельзя. Аварийное прерывание клиентской команды может оставить server-side `pg_sleep` до окончания заданного ограниченного времени; проверьте состояние блокировки. `python3 student-tests/slo_ready_incident.py` проверяет отсутствие незавершённых заданий, останавливает worker, принимает восемь синтетических файлов и восстанавливает worker в `finally`. До запуска убедитесь, что API и worker свободны. Результат — [evidence/slo-ready-incident.json](../evidence/slo-ready-incident.json). Снятие alerts при малом количестве новых событий может означать недостаток событий в коротком окне, а не доказательство выполнения 24-часового SLO. @@ -39,3 +39,7 @@ docker run --rm -v "$PWD:/workspace:ro" --entrypoint promtool prom/prometheus:v3 ``` В runbook зафиксируйте фазу, UTC начала/окончания, первый firing и первое наблюдение после исчезновения alert. Первый и последний интервалы ограничены частотой опроса, а отсутствие ложного firing в одном baseline не оценивает долгосрочный false-positive rate. + +## Проверка другим студентом + +Проверяющий не получает команду инъекции заранее. Автор запускает один из двух ограниченных сценариев; проверяющий получает время и имя alert, затем проходит шаги выше. В отдельный протокол записывают: имя проверяющего, UTC, запрос PromQL/LogQL, trace ID, медленный span или отсутствие consumer, диагноз, действие восстановления и свидетельство resolved. Совпадение диагноза с инъекцией проверяют после записи. Пока такого заполненного протокола нет, независимое прохождение runbook не подтверждено. diff --git a/evidence/pre-review/ready-scenario-failed.json b/evidence/pre-review/ready-scenario-failed.json new file mode 100644 index 0000000..feaff5a --- /dev/null +++ b/evidence/pre-review/ready-scenario-failed.json @@ -0,0 +1,300 @@ +{ + "samples": [ + { + "time": "2026-11-06T07:03:03.661461+00:00", + "monotonic": 51955.506616, + "phase": "baseline", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:09.013918+00:00", + "monotonic": 51960.859020333, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:14.036654+00:00", + "monotonic": 51965.881710833, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:19.064110+00:00", + "monotonic": 51970.909114958, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:24.098276+00:00", + "monotonic": 51975.943249958, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:29.150984+00:00", + "monotonic": 51980.9959115, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:34.199153+00:00", + "monotonic": 51986.044040208, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:39.245809+00:00", + "monotonic": 51991.09063275, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:44.282804+00:00", + "monotonic": 51996.127578458, + "phase": "worker_stopped", + "firing": [], + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:49.317995+00:00", + "monotonic": 52001.162700583, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:54.345636+00:00", + "monotonic": 52006.190306166, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:03:59.376176+00:00", + "monotonic": 52011.220777958, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:04.416058+00:00", + "monotonic": 52016.260635708, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:09.450875+00:00", + "monotonic": 52021.295390541, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:14.481279+00:00", + "monotonic": 52026.325748, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:19.519443+00:00", + "monotonic": 52031.36385225, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:24.550837+00:00", + "monotonic": 52036.395194208, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:29.576411+00:00", + "monotonic": 52041.420720208, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:34.607663+00:00", + "monotonic": 52046.451919791, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:39.633257+00:00", + "monotonic": 52051.477466458, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + }, + { + "time": "2026-11-06T07:04:44.663734+00:00", + "monotonic": 52056.50792775, + "phase": "worker_stopped", + "firing": [], + "all_firing": [ + "MessengerQueueLag" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 9.0, + "bad": 8.0 + } + ], + "jobs": 8, + "accepted_at": { + "time": "2026-11-06T07:03:09.009364+00:00", + "monotonic": 51960.854465416 + } +} diff --git a/evidence/pre-review/slo-analysis.json b/evidence/pre-review/slo-analysis.json new file mode 100644 index 0000000..180188b --- /dev/null +++ b/evidence/pre-review/slo-analysis.json @@ -0,0 +1,59 @@ +{ + "sampling_interval_seconds": 5, + "monitor_errors": 0, + "phases": { + "burst": { + "observed_lock_seconds": 75.033, + "alerts": { + "MessengerApiBurnFast": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:30:49.560423+00:00", + "first_firing_after_incident": false, + "detection_seconds": 55.088, + "first_clear": "2026-11-05T19:31:24.687321+00:00", + "recovery_seconds": 15.181 + }, + "MessengerApiBurnSlow": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:31:14.651231+00:00", + "first_firing_after_incident": true, + "detection_seconds": 80.179, + "first_clear": "2026-11-05T19:31:24.687321+00:00", + "recovery_seconds": 15.181 + } + } + }, + "sustained": { + "observed_lock_seconds": 180.134, + "alerts": { + "MessengerApiBurnFast": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:33:40.096081+00:00", + "first_firing_after_incident": false, + "detection_seconds": 50.278, + "first_clear": "2026-11-05T19:36:10.387240+00:00", + "recovery_seconds": 10.648 + }, + "MessengerApiBurnSlow": { + "active_before_lock": false, + "first_firing": "2026-11-05T19:34:20.245337+00:00", + "first_firing_after_incident": false, + "detection_seconds": 90.428, + "first_clear": "2026-11-05T19:37:53.534744+00:00", + "recovery_seconds": 30.724 + } + } + } + }, + "timing_quality": { + "max_wall_monotonic_disagreement_seconds": 83.075, + "max_observed_sampling_gap_seconds": 5.029, + "valid_for_elapsed_time_claims": false, + "note": "Disagreement does not identify its cause; rerun if invalid. Precision is bounded by actual gaps, not the configured sleep." + }, + "baseline": { + "samples": 13, + "samples_with_burn_alert": 0, + "scope": "one observed healthy baseline, not a long-term false-positive rate" + } +} diff --git a/evidence/pre-review/slo-incidents.json b/evidence/pre-review/slo-incidents.json new file mode 100644 index 0000000..742ed8f --- /dev/null +++ b/evidence/pre-review/slo-incidents.json @@ -0,0 +1,1170 @@ +{ + "description": "controlled DB table lock; three synthetic clients; 5s alert sampling", + "periods": { + "burst": { + "lock_observed": { + "time": "2026-11-05T19:29:54.472744+00:00", + "monotonic": 43527.079238041, + "command_started": { + "time": "2026-11-05T19:29:54.163828+00:00", + "monotonic": 43526.770364958 + } + }, + "command_completed": { + "time": "2026-11-05T19:31:09.505932+00:00", + "monotonic": 43602.112587416 + } + }, + "sustained": { + "lock_observed": { + "time": "2026-11-05T19:32:49.817996+00:00", + "monotonic": 43702.424801208, + "command_started": { + "time": "2026-11-05T19:32:49.516057+00:00", + "monotonic": 43702.122908708 + } + }, + "command_completed": { + "time": "2026-11-05T19:35:59.744503+00:00", + "monotonic": 43882.558427291 + } + } + }, + "samples": [ + { + "time": "2026-11-05T19:28:54.176617+00:00", + "monotonic": 43466.783021166, + "phase": "baseline", + "counts": { + "ok": 1, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:28:59.184768+00:00", + "monotonic": 43471.791177208, + "phase": "baseline", + "counts": { + "ok": 15, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:04.205356+00:00", + "monotonic": 43476.8117975, + "phase": "baseline", + "counts": { + "ok": 30, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:09.226269+00:00", + "monotonic": 43481.832718708, + "phase": "baseline", + "counts": { + "ok": 45, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:14.243046+00:00", + "monotonic": 43486.849499708, + "phase": "baseline", + "counts": { + "ok": 60, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:19.249330+00:00", + "monotonic": 43491.855772208, + "phase": "baseline", + "counts": { + "ok": 75, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:24.265281+00:00", + "monotonic": 43496.871750375, + "phase": "baseline", + "counts": { + "ok": 90, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:29.284910+00:00", + "monotonic": 43501.891384583, + "phase": "baseline", + "counts": { + "ok": 105, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:34.295325+00:00", + "monotonic": 43506.901793458, + "phase": "baseline", + "counts": { + "ok": 120, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:39.306441+00:00", + "monotonic": 43511.91291825, + "phase": "baseline", + "counts": { + "ok": 135, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:44.313768+00:00", + "monotonic": 43516.920254333, + "phase": "baseline", + "counts": { + "ok": 150, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:49.331915+00:00", + "monotonic": 43521.938423708, + "phase": "baseline", + "counts": { + "ok": 165, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:54.351411+00:00", + "monotonic": 43526.957913708, + "phase": "baseline", + "counts": { + "ok": 180, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:29:59.372201+00:00", + "monotonic": 43531.978732458, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 0 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:04.386019+00:00", + "monotonic": 43536.992552291, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 3 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:09.401930+00:00", + "monotonic": 43542.0084625, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 6 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:14.419029+00:00", + "monotonic": 43547.025585291, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 9 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:19.433563+00:00", + "monotonic": 43552.040130916, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 12 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:24.450146+00:00", + "monotonic": 43557.05672225, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 15 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:29.473233+00:00", + "monotonic": 43562.079822583, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 15 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:34.493955+00:00", + "monotonic": 43567.1005505, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 18 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:39.519370+00:00", + "monotonic": 43572.125976166, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 21 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:44.535535+00:00", + "monotonic": 43577.142128416, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 24 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:30:49.560423+00:00", + "monotonic": 43582.167042875, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 27 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:30:54.583572+00:00", + "monotonic": 43587.190215125, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 30 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:30:59.600250+00:00", + "monotonic": 43592.206889708, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 30 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:31:04.616531+00:00", + "monotonic": 43597.223167708, + "phase": "burst", + "counts": { + "ok": 180, + "failed": 33 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:31:09.632643+00:00", + "monotonic": 43602.239279708, + "phase": "burst_recovery", + "counts": { + "ok": 183, + "failed": 36 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:31:14.651231+00:00", + "monotonic": 43607.257891625, + "phase": "burst_recovery", + "counts": { + "ok": 198, + "failed": 36 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:31:19.663273+00:00", + "monotonic": 43612.269926416, + "phase": "burst_recovery", + "counts": { + "ok": 211, + "failed": 36 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:31:24.687321+00:00", + "monotonic": 43617.294002291, + "phase": "burst_recovery", + "counts": { + "ok": 225, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:29.708468+00:00", + "monotonic": 43622.315158041, + "phase": "burst_recovery", + "counts": { + "ok": 240, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:34.728166+00:00", + "monotonic": 43627.334856708, + "phase": "burst_recovery", + "counts": { + "ok": 255, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:39.733767+00:00", + "monotonic": 43632.340459958, + "phase": "burst_recovery", + "counts": { + "ok": 270, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:44.755480+00:00", + "monotonic": 43637.36218675, + "phase": "burst_recovery", + "counts": { + "ok": 285, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:49.769677+00:00", + "monotonic": 43642.376390333, + "phase": "burst_recovery", + "counts": { + "ok": 300, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:54.778133+00:00", + "monotonic": 43647.384860875, + "phase": "burst_recovery", + "counts": { + "ok": 315, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:31:59.788234+00:00", + "monotonic": 43652.394963833, + "phase": "burst_recovery", + "counts": { + "ok": 330, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:04.797259+00:00", + "monotonic": 43657.403998, + "phase": "burst_recovery", + "counts": { + "ok": 345, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:09.813581+00:00", + "monotonic": 43662.42033325, + "phase": "burst_recovery", + "counts": { + "ok": 360, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:14.823961+00:00", + "monotonic": 43667.430715458, + "phase": "burst_recovery", + "counts": { + "ok": 375, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:19.834831+00:00", + "monotonic": 43672.441596833, + "phase": "burst_recovery", + "counts": { + "ok": 390, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:24.847866+00:00", + "monotonic": 43677.454636958, + "phase": "burst_recovery", + "counts": { + "ok": 405, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:29.862747+00:00", + "monotonic": 43682.469528375, + "phase": "burst_recovery", + "counts": { + "ok": 417, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:34.870564+00:00", + "monotonic": 43687.47735275, + "phase": "burst_recovery", + "counts": { + "ok": 432, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:39.880308+00:00", + "monotonic": 43692.487106, + "phase": "burst_recovery", + "counts": { + "ok": 447, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:44.899942+00:00", + "monotonic": 43697.506751, + "phase": "burst_recovery", + "counts": { + "ok": 462, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:49.916853+00:00", + "monotonic": 43702.523669458, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:54.942437+00:00", + "monotonic": 43707.549288958, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 36 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:32:59.957394+00:00", + "monotonic": 43712.564228083, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 39 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:04.970687+00:00", + "monotonic": 43717.577532333, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 42 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:09.993536+00:00", + "monotonic": 43722.600420875, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 45 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:15.009372+00:00", + "monotonic": 43727.616250583, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 48 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:20.031828+00:00", + "monotonic": 43732.638718, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 51 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:25.051997+00:00", + "monotonic": 43737.658880125, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 51 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:30.067799+00:00", + "monotonic": 43742.674685791, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 54 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:35.075453+00:00", + "monotonic": 43747.682349166, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 57 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:33:40.096081+00:00", + "monotonic": 43752.703001416, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 60 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:33:45.119089+00:00", + "monotonic": 43757.7260235, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 63 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:33:50.133709+00:00", + "monotonic": 43762.740644916, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 66 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:33:55.161233+00:00", + "monotonic": 43767.768187291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 66 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:00.176785+00:00", + "monotonic": 43772.783745208, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 69 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:05.196858+00:00", + "monotonic": 43777.803812291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 72 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:10.212935+00:00", + "monotonic": 43782.819901041, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 75 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:15.231470+00:00", + "monotonic": 43787.838454541, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 78 + }, + "firing": [ + "MessengerApiBurnFast" + ] + }, + { + "time": "2026-11-05T19:34:20.245337+00:00", + "monotonic": 43792.852329875, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 81 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:25.257984+00:00", + "monotonic": 43797.864980958, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 81 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:30.282024+00:00", + "monotonic": 43802.889036875, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 84 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:35.303328+00:00", + "monotonic": 43807.910348833, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 87 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:40.328342+00:00", + "monotonic": 43812.935370625, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 90 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:45.343604+00:00", + "monotonic": 43817.950633291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 93 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:50.359471+00:00", + "monotonic": 43822.966518166, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 96 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:34:55.367578+00:00", + "monotonic": 43827.974616416, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 96 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:00.386215+00:00", + "monotonic": 43832.993284208, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 99 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:05.400579+00:00", + "monotonic": 43838.0076475, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 102 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:10.422157+00:00", + "monotonic": 43843.0292365, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 105 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:15.442075+00:00", + "monotonic": 43848.049173416, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 108 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:20.452042+00:00", + "monotonic": 43853.05914025, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 111 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:25.461182+00:00", + "monotonic": 43858.068280916, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 111 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:30.480197+00:00", + "monotonic": 43863.087333625, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 114 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:35.504075+00:00", + "monotonic": 43868.111188291, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 117 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:40.521486+00:00", + "monotonic": 43873.129961791, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 120 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:35:55.348285+00:00", + "monotonic": 43878.158763833, + "phase": "sustained", + "counts": { + "ok": 477, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:00.359177+00:00", + "monotonic": 43883.173501041, + "phase": "sustained_recovery", + "counts": { + "ok": 480, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:05.367876+00:00", + "monotonic": 43888.184996, + "phase": "sustained_recovery", + "counts": { + "ok": 495, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:10.387240+00:00", + "monotonic": 43893.206441208, + "phase": "sustained_recovery", + "counts": { + "ok": 510, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:15.397936+00:00", + "monotonic": 43898.218563625, + "phase": "sustained_recovery", + "counts": { + "ok": 525, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:36:20.421727+00:00", + "monotonic": 43903.243449875, + "phase": "sustained_recovery", + "counts": { + "ok": 540, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:37:48.510428+00:00", + "monotonic": 43908.257572375, + "phase": "sustained_recovery", + "counts": { + "ok": 555, + "failed": 123 + }, + "firing": [ + "MessengerApiBurnSlow" + ] + }, + { + "time": "2026-11-05T19:37:53.534744+00:00", + "monotonic": 43913.282484708, + "phase": "sustained_recovery", + "counts": { + "ok": 570, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:37:58.549296+00:00", + "monotonic": 43918.29743975, + "phase": "sustained_recovery", + "counts": { + "ok": 585, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:03.567430+00:00", + "monotonic": 43923.315891, + "phase": "sustained_recovery", + "counts": { + "ok": 600, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:08.593041+00:00", + "monotonic": 43928.341731291, + "phase": "sustained_recovery", + "counts": { + "ok": 615, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:13.607305+00:00", + "monotonic": 43933.356177, + "phase": "sustained_recovery", + "counts": { + "ok": 630, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:18.617266+00:00", + "monotonic": 43938.366246166, + "phase": "sustained_recovery", + "counts": { + "ok": 642, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:23.636630+00:00", + "monotonic": 43943.385724875, + "phase": "sustained_recovery", + "counts": { + "ok": 657, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:28.654169+00:00", + "monotonic": 43948.403359125, + "phase": "sustained_recovery", + "counts": { + "ok": 672, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:33.670193+00:00", + "monotonic": 43953.419418, + "phase": "sustained_recovery", + "counts": { + "ok": 687, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:38.676034+00:00", + "monotonic": 43958.425318541, + "phase": "sustained_recovery", + "counts": { + "ok": 702, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:43.699279+00:00", + "monotonic": 43963.448615416, + "phase": "sustained_recovery", + "counts": { + "ok": 717, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:48.718747+00:00", + "monotonic": 43968.468122083, + "phase": "sustained_recovery", + "counts": { + "ok": 732, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:53.742289+00:00", + "monotonic": 43973.491697375, + "phase": "sustained_recovery", + "counts": { + "ok": 747, + "failed": 123 + }, + "firing": [] + }, + { + "time": "2026-11-05T19:38:58.757203+00:00", + "monotonic": 43978.50662725, + "phase": "sustained_recovery", + "counts": { + "ok": 762, + "failed": 123 + }, + "firing": [] + } + ], + "end_time": "2026-11-05T19:39:02.820853+00:00", + "final_counts": { + "ok": 774, + "failed": 123 + } +} diff --git a/evidence/pre-review/slo-ready-incident.json b/evidence/pre-review/slo-ready-incident.json new file mode 100644 index 0000000..5eff503 --- /dev/null +++ b/evidence/pre-review/slo-ready-incident.json @@ -0,0 +1,211 @@ +{ + "samples": [ + { + "time": "2026-11-06T06:18:43.857552+00:00", + "monotonic": 49295.73000975, + "phase": "baseline", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:18:49.195942+00:00", + "monotonic": 49301.068345333, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:18:54.211236+00:00", + "monotonic": 49306.083585125, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:18:59.238528+00:00", + "monotonic": 49311.110847333, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:04.276851+00:00", + "monotonic": 49316.149107833, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:09.316299+00:00", + "monotonic": 49321.18853025, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:14.361014+00:00", + "monotonic": 49326.233190833, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:19.399292+00:00", + "monotonic": 49331.271419, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:24.439389+00:00", + "monotonic": 49336.31146675, + "phase": "worker_stopped", + "firing": [], + "eligible": 9.0, + "bad": 4.0 + }, + { + "time": "2026-11-06T06:19:29.469853+00:00", + "monotonic": 49341.341864875, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:35.732760+00:00", + "monotonic": 49347.604713666, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:40.766904+00:00", + "monotonic": 49352.638829583, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:45.800827+00:00", + "monotonic": 49357.672704333, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:50.839444+00:00", + "monotonic": 49362.711278166, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:19:55.882108+00:00", + "monotonic": 49367.753864791, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:00.935125+00:00", + "monotonic": 49372.806871, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:05.976123+00:00", + "monotonic": 49377.847798625, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:11.006201+00:00", + "monotonic": 49382.877811166, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:16.034665+00:00", + "monotonic": 49387.906272875, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:21.073601+00:00", + "monotonic": 49392.945128916, + "phase": "recovery", + "firing": [ + "MessengerReadyBurnSlow" + ], + "eligible": 17.0, + "bad": 12.0 + }, + { + "time": "2026-11-06T06:20:26.112334+00:00", + "monotonic": 49397.983810916, + "phase": "recovery", + "firing": [], + "eligible": 17.0, + "bad": 12.0 + } + ], + "jobs": 8, + "accepted_at": { + "time": "2026-11-06T06:18:49.190891+00:00", + "monotonic": 49301.063291333 + }, + "worker_started": { + "time": "2026-11-06T06:19:31.951591+00:00", + "monotonic": 49343.823643916 + }, + "all_ready": { + "time": "2026-11-06T06:19:35.725614+00:00", + "monotonic": 49347.597564333 + }, + "result": "PASS" +} diff --git a/evidence/review-checks.json b/evidence/review-checks.json new file mode 100644 index 0000000..cd47500 --- /dev/null +++ b/evidence/review-checks.json @@ -0,0 +1,109 @@ +{ + "utc": "2026-11-06T07:13:15.774667+00:00", + "checks": [ + { + "command": [ + "make", + "check" + ], + "exit_code": 0, + "output": "python3 scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n" + }, + { + "command": [ + "make", + "validate", + "PYTHON=.venv/bin/python" + ], + "exit_code": 0, + "output": ".venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\n" + }, + { + "command": [ + "docker", + "compose", + "config", + "--quiet" + ], + "exit_code": 0, + "output": "" + }, + { + "command": [ + "docker", + "run", + "--rm", + "-v", + "/Users/garfieldmk/Desktop/study/highLoad/mtusi2026lab6:/workspace:ro", + "--entrypoint", + "promtool", + "prom/prometheus:v3.2.1", + "test", + "rules", + "/workspace/student-tests/alerts.test.yaml", + "/workspace/student-tests/burn_rules.test.yaml", + "/workspace/student-tests/telemetry_alerts.test.yaml" + ], + "exit_code": 0, + "output": " SUCCESS\n\n SUCCESS\n\n SUCCESS\n\n" + }, + { + "command": [ + "make", + "test" + ], + "exit_code": 0, + "output": "python3 tests/smoke.py --base-url \"http://localhost:8080\" \nPASS: lab6 public API scenario; infrastructure and grading evidence remain separate.\n" + }, + { + "command": [ + "python3", + "student-tests/check_dashboard.py" + ], + "exit_code": 0, + "output": "PASS: 27 dashboard queries; empty targets: 5\n" + }, + { + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "stdin": "student-tests/telemetry_unit.py", + "exit_code": 0, + "output": "......\n----------------------------------------------------------------------\nRan 6 tests in 0.097s\n\nOK\n" + }, + { + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "stdin": "student-tests/pipeline_critical.py", + "exit_code": 0, + "output": "2026-11-06 07:13:14,688 WARNING worker staging cleanup deferred attachment_id=e9a74014-aaa3-4d97-9012-baf9611cc58b\n.........\n----------------------------------------------------------------------\nRan 9 tests in 0.010s\n\nOK\n" + }, + { + "command": [ + "docker", + "compose", + "exec", + "-T", + "api-a", + "/app/.venv/bin/python", + "-" + ], + "stdin": "student-tests/slo_commit_integration.py", + "exit_code": 0, + "output": "PASS: commit boundaries; immutable good/bad cohorts; late completion remains bad\n" + } + ] +} diff --git a/evidence/review-dashboard.json b/evidence/review-dashboard.json new file mode 100644 index 0000000..e02837c --- /dev/null +++ b/evidence/review-dashboard.json @@ -0,0 +1,137 @@ +[ + { + "panel": "API RPS", + "query": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))", + "series": 1 + }, + { + "panel": "API error ratio", + "query": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])) > 0", + "series": 0 + }, + { + "panel": "API latency p50 / p95 / p99", + "query": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "series": 1 + }, + { + "panel": "API latency p50 / p95 / p99", + "query": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "series": 1 + }, + { + "panel": "API latency p50 / p95 / p99", + "query": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "series": 1 + }, + { + "panel": "Jobs by outcome", + "query": "sum by (outcome) (rate(messenger_jobs_total[1m]))", + "series": 2 + }, + { + "panel": "Job duration p95", + "query": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "series": 1 + }, + { + "panel": "Acceptance to ready p95", + "query": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_ready_duration_seconds_bucket[5m])))", + "series": 1 + }, + { + "panel": "Queued jobs", + "query": "max(messenger_queue_depth{service_name=\"messenger-outbox\"})", + "series": 1 + }, + { + "panel": "Oldest queued age", + "query": "max(messenger_queue_age_seconds{service_name=\"messenger-outbox\"})", + "series": 1 + }, + { + "panel": "DB pool used / capacity / waiting", + "query": "messenger_pool_used{service_name=\"messenger-api\"}", + "series": 2 + }, + { + "panel": "DB pool used / capacity / waiting", + "query": "messenger_pool_capacity{service_name=\"messenger-api\"}", + "series": 2 + }, + { + "panel": "DB pool used / capacity / waiting", + "query": "messenger_pool_waiting{service_name=\"messenger-api\"}", + "series": 2 + }, + { + "panel": "Worker busy / capacity", + "query": "messenger_worker_busy{service_name=\"messenger-worker\"}", + "series": 2 + }, + { + "panel": "Worker busy / capacity", + "query": "messenger_worker_capacity{service_name=\"messenger-worker\"}", + "series": 2 + }, + { + "panel": "Process CPU", + "query": "sum by (service_name, service_instance_id) (rate(messenger_process_cpu_seconds_total[1m]))", + "series": 6 + }, + { + "panel": "Process resident memory", + "query": "messenger_process_memory_bytes", + "series": 5 + }, + { + "panel": "Collector failed / dropped spans per second", + "query": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[1m]))", + "series": 1 + }, + { + "panel": "Collector failed / dropped spans per second", + "query": "sum by (exporter) (rate(otelcol_exporter_enqueue_failed_spans_total[1m]))", + "series": 1 + }, + { + "panel": "API SLI: successful ratio over 24h", + "query": "1 - (sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[24h])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[24h])),1) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[24h])) > 0", + "series": 1 + }, + { + "panel": "API SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[20s])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[20s])) > 0", + "series": 0 + }, + { + "panel": "API SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[1m])) > 0", + "series": 0 + }, + { + "panel": "API SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[5m])), 1)) / 0.01) and sum(increase(messenger_http_requests_total{route!~\"/health/.*\"}[5m])) > 0", + "series": 1 + }, + { + "panel": "READY SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[20s])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[20s])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[20s])) > 0", + "series": 0 + }, + { + "panel": "READY SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[1m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[1m])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[1m])) > 0", + "series": 0 + }, + { + "panel": "READY SLO burn rate (budget 1%)", + "query": "(((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[5m])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[5m])), 1)) / 0.01) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[5m])) > 0", + "series": 1 + }, + { + "panel": "Accepted jobs ready within 30 s — 24h (matured cohort)", + "query": "1 - ((sum(increase(messenger_slo_ready_bad_total{service_name=\"messenger-outbox\"}[24h])) or vector(0)) / clamp_min(sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[24h])), 1)) and sum(increase(messenger_slo_ready_eligible_total{service_name=\"messenger-outbox\"}[24h])) > 0", + "series": 1 + } +] diff --git a/evidence/review-db-trace.json b/evidence/review-db-trace.json new file mode 100644 index 0000000..16cffb9 --- /dev/null +++ b/evidence/review-db-trace.json @@ -0,0 +1,909 @@ +{ + "captured_at": "2026-11-06T06:54:49.009234+00:00", + "query": "{service_name=\"messenger-api\"} | status=\"503\"", + "trace_id": "b9b17078824f1cc4e21a4434e129efc9", + "logs": { + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948029625964635", + "request_id": "51734099-b47b-433a-abb2-ce06a64c1478", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "151f102afc35d89a", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "b9b17078824f1cc4e21a4434e129efc9" + }, + "values": [ + [ + "1793948029625911040", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948029626627968", + "request_id": "55344573-05c5-474e-abd5-b260b03a332f", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "60f1eaa06a9e9785", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "fc93bdd340de079da209246d67f8a561" + }, + "values": [ + [ + "1793948029626592512", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948035645779263", + "request_id": "5b6032db-2e31-4f04-a385-456ebffebfdc", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "9a650ee7231a02bf", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "1c58397c3ec8085b1e27609b27ae520c" + }, + "values": [ + [ + "1793948035645726464", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948035645787846", + "request_id": "8acd3330-9d9a-4c2c-8d84-f0e4ff441d4e", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "b0df0e4132d5949b", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "2498be54dd8888579ad47c9fb167cf18" + }, + "values": [ + [ + "1793948035645739520", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948035646862221", + "request_id": "5b0a1b31-a671-47a9-86e2-758ccb5dc675", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "561098ed9ea394b0", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "c94bf7e1d16e7c55357478fc8a25648e" + }, + "values": [ + [ + "1793948035646823168", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948041671332960", + "request_id": "7aff0420-ff29-4c90-96e3-cbc5a2532e30", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "6f8d3db9589f262e", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "2c73801c22050e58905cf9d73060ff43" + }, + "values": [ + [ + "1793948041671150080", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948041671636210", + "request_id": "7f3e51d9-3057-47a6-9896-d53c1b52d498", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "05853b814f44dd57", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "134beebdc0479e59dacd0d3a91c6824c" + }, + "values": [ + [ + "1793948041671464960", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948041672626877", + "request_id": "fd9f4531-0d77-4abf-9232-6e621c2dc69a", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "28115ec0c6bf870b", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "d8f558f27d33ee11545f0c6c782092ab" + }, + "values": [ + [ + "1793948041672583424", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948047681994963", + "request_id": "c2dd6234-4f16-40a6-9ac8-adecbe4a2959", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "9c95cf9971a250f6", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "a4d252574094985c29f71f4c76ad265e" + }, + "values": [ + [ + "1793948047681970944", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948047684209629", + "request_id": "d179ce27-8330-4b45-94bb-ade25970ac20", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "d379e86b1b590491", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "a161022e47d517d700bc98c8f1a61dff" + }, + "values": [ + [ + "1793948047684182528", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948047684229754", + "request_id": "f1bba5fc-b784-4326-b311-89e135e9cbf2", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "85dc24e82075299c", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "300e5c652019f265efd75db929f5b52b" + }, + "values": [ + [ + "1793948047684212736", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948053706657382", + "request_id": "d9a079b7-3177-43f1-aadd-29fd78496ea9", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "76b413ac9de41425", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "29a9abc1e5f3b9092d58b881f0736412" + }, + "values": [ + [ + "1793948053706575104", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948053706669174", + "request_id": "014723e9-26db-496d-883b-9b2db8bcd340", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "d93bf6199b8546fc", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "9088b24f0cd2fd1a018e0db423799194" + }, + "values": [ + [ + "1793948053706575360", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948053707987632", + "request_id": "353b4848-174a-4591-97bf-6d9f617fd313", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "f41d3c6bcba07369", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "f7c3d5d493bbd9df21991623089fbdf0" + }, + "values": [ + [ + "1793948053707940096", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948059729947552", + "request_id": "6c9a13a3-4311-4f0d-8076-72c3d1e2f9f4", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "f4ddfd85dcde62ee", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "180f3189cf8a2830a6180baf8e0fcd08" + }, + "values": [ + [ + "1793948059729844992", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948059730063635", + "request_id": "fd704a6f-49b8-4fcd-93bd-12946f9a4244", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "2a4cfc787931f6ed", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "b5d08507f638c052061e9b2b03323048" + }, + "values": [ + [ + "1793948059729981184", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948059730736968", + "request_id": "47ea9ee4-dbd2-4fe1-a369-90a43ebf49b6", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "9c11c85b2900685e", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "8637ac13af6d5a61d8e3a3ea26d3968a" + }, + "values": [ + [ + "1793948059730694912", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948065753870555", + "request_id": "52575c60-c97b-4ebc-a214-f762c399e9ac", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "f13c6a8f6a55fe64", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "ffe846cc93dfae2f682b097e9a304a57" + }, + "values": [ + [ + "1793948065753797120", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948065753873221", + "request_id": "2d6a57dc-1934-4403-bb8a-69f40d20049d", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "f1c146213339", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "ff28ad796a5c16fc", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "3631b8f5fb534b466f0200d125ba50c2" + }, + "values": [ + [ + "1793948065753789696", + "http.request" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948065755343888", + "request_id": "cc3aecc7-1fe0-43b9-9f4d-46cd5f9dd63c", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger.telemetry", + "service_instance_id": "d0867cbe3aca", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "93d157788c8db59a", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "78ff7789eadd30821784a80d298a9371" + }, + "values": [ + [ + "1793948065755291136", + "http.request" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 55251140, + "linesProcessedPerSecond": 336897, + "totalBytesProcessed": 513976, + "totalLinesProcessed": 3134, + "execTime": 0.009303, + "queueTime": 8.4e-05, + "subqueries": 0, + "totalEntriesReturned": 20, + "splits": 0, + "shards": 1, + "totalPostFilterLines": 36, + "totalStructuredMetadataBytesProcessed": 476368 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 1, + "totalChunksMatched": 4, + "totalBatches": 1, + "totalLinesSent": 20, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": true, + "chunk": { + "headChunkBytes": 513976, + "headChunkLines": 3134, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 36, + "headChunkStructuredMetadataBytes": 476368, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 122375, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false + } + } + } + }, + "trace": { + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "f1c146213339" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "ubFweIJPHMTiGkQ04SnvyQ==", + "spanId": "6fTEyNCjCc8=", + "parentSpanId": "FR8QKvw12Jo=", + "name": "postgres.user_for_token", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948024619601966", + "endTimeUnixNano": "1793948029625443968", + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "ubFweIJPHMTiGkQ04SnvyQ==", + "spanId": "FR8QKvw12Jo=", + "name": "POST /api/v1/chats/{chat_id}/messages", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1793948024616785508", + "endTimeUnixNano": "1793948029626105093", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/messages" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + } + ] + } + ] + } + ] + } +} diff --git a/evidence/review-observability.md b/evidence/review-observability.md new file mode 100644 index 0000000..0ac8c85 --- /dev/null +++ b/evidence/review-observability.md @@ -0,0 +1,29 @@ +# Повторная проверка 2026-11-06 + +Команды и stdout/exit codes коротких проверок: [review-checks.json](review-checks.json). В Docker Python 3.12 прошли 6 SDK-тестов, 9 критических проверок pipeline и интеграционная проверка commit-границ во временной таблице. Публичный smoke выполнялся при включённом OTLP. Все 27 запросов dashboard корректны; пустые ready burn ряды при отсутствии новых событий не заменены искусственным значением «здоров» — [review-dashboard.json](review-dashboard.json). + +## Связь API и worker с retry + +[Сценарий](../student-tests/retry_evidence.py) включает fault после S3 PUT, затем возвращает обычную конфигурацию worker в finally. [Результат](review-retry-job.json): ready через 61,421 с, `attempt=2`, один рестарт контейнера. [Trace и логи](review-retry-trace.json) содержат 21 span, 2 producer и 2 consumer, операции S3 и завершения DB. [Проверка дерева](review-retry-summary.json) подтверждает одного API-предка для producer и producer-родителей consumer, общий trace ID и логи обоих сервисов. Сохранённый контекст находится внутри `postgres.create_attachment`, поэтому producer связан с API через DB span, а не обязан быть прямым ребёнком SERVER. Незавершённый span аварийной попытки может не экспортироваться при `os._exit`; полнота упавшей попытки не заявляется. + +## Два инцидента приложения + +- БД: [сырые пробы](slo-incidents.json) включают `MessengerHighErrorRate` firing и его отсутствие в конце. [Коррелированный лог и trace](review-db-trace.json) показывают 503, DB span 5,006 с и HTTP span 5,009 с; времена — разность timestamp полей /10⁹. Причина диагностируется по DB span. [Анализ](slo-analysis.json) сохраняет detection/recovery и качество времени. +- Worker: [пробы](slo-ready-incident.json) показывают очередь, ReadyBurnFast/Slow и QueueLag, затем depth/age = 0 и отсутствие alerts. Все 8 новых заданий нарушили срок: приращение eligible и bad равно 8. Fast исчез до восстановления worker вследствие короткого окна/minimum volume; это ограничение правил, а не доказательство здоровья. Числа и объяснение — [slo-track.md](slo-track.md). + +## Недоступность Tempo + +[Сценарий](../student-tests/tempo_outage.py), [метрики и память](review-tempo-outage.json), [нагрузка](review-tempo-load.json). Tempo остановлен, smoke прошёл. Затем concurrency=4 за 75,006 с дал 40 628 успешных запросов и 0 ошибок; успешные RPS = `40628 / 75,006 ≈ 541,663` (генератор сообщает 541,661 по неокруглённому времени). Это closed-loop опыт, а не максимальная производительность. + +Collector не пересоздавался: ID в начале и конце одинаков. По сохранённым Prometheus-пробам: + +- Очередь выросла от 0 до лимита 128, после восстановления опустела до 0. +- Память по `docker stats` — 114,6 MiB до опыта; максимум сохранённых проб — 128,8 MiB при лимите 256 MiB. Это выборочные наблюдения ограниченного опыта, не доказательство ограниченности на произвольном времени. +- Дельта `enqueue_failed_spans_total` — 57 268; дельта `send_failed_spans_total` — 772; дельта `sent_spans_total` — 64 860. Каждая дельта = последняя проба − baseline. Первые две величины характеризуют разные этапы ошибок экспорта и не складываются в число уникальных потерянных spans. +- Через первые 25 с после запуска Tempo очередь ещё была полной. Продолжение того же замера подтвердило опустошение; исходные пробы сохранены. Сценарий исправлен: теперь проверяет фактическое опустошение с пределом ожидания 90 с, а не только успешный запуск контейнера. + +Ограниченная очередь допускает потерю traces при переполнении; доступность API в данном опыте сохранена. Повторный запуск не обязан дать те же абсолютные числа: исходная нагрузка, моменты scrape и размер batches влияют на результат. + +## Сохранность и отказ реплики + +[Публичные тесты](review-resilience.json) persistence и failover завершились PASS; runner восстановил обе API-реплики. [Повторный запрос Tempo](review-tempo-persistence.json) после stop/start вернул те же 21 span трассы, записанной до отказа. Конфигурация persistent volumes находится в Compose; удаление volumes в этих сценариях не выполнялось. diff --git a/evidence/review-resilience.json b/evidence/review-resilience.json new file mode 100644 index 0000000..1bdf90e --- /dev/null +++ b/evidence/review-resilience.json @@ -0,0 +1,27 @@ +{ + "utc": "2026-11-06T07:15:28.687307+00:00", + "checks": [ + { + "command": [ + "make", + "test-persistence", + "ACTION_SCRIPT=scripts/restart.sh" + ], + "exit_code": 0, + "output": "python3 tests/resilience.py --mode persistence --action-script \"scripts/restart.sh\" --base-url \"http://localhost:8080\" \nExecuting explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.\n" + }, + { + "command": [ + "make", + "test-failover", + "ACTION_SCRIPT=scripts/stop-one.sh" + ], + "exit_code": 0, + "output": "python3 tests/resilience.py --mode failover --action-script \"scripts/stop-one.sh\" --base-url \"http://localhost:8080\" \nExecuting explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 112, \"failed_attempts\": 1, \"first_recovery_seconds\": 2.561}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.\n" + }, + { + "restore_exit_code": 0, + "output": " Container mtusi2026lab6-api-a-1 Running \n Container mtusi2026lab6-outbox-1 Running \n Container mtusi2026lab6-worker-1 Running \n Container mtusi2026lab6-api-b-1 Starting \n Container mtusi2026lab6-api-b-1 Started \n Container mtusi2026lab6-outbox-1 Waiting \n Container mtusi2026lab6-api-b-1 Waiting \n Container mtusi2026lab6-worker-1 Waiting \n Container mtusi2026lab6-api-a-1 Waiting \n Container mtusi2026lab6-outbox-1 Healthy \n Container mtusi2026lab6-api-a-1 Healthy \n Container mtusi2026lab6-worker-1 Healthy \n Container mtusi2026lab6-api-b-1 Healthy \n" + } + ] +} diff --git a/evidence/review-retry-job.json b/evidence/review-retry-job.json new file mode 100644 index 0000000..4f95172 --- /dev/null +++ b/evidence/review-retry-job.json @@ -0,0 +1,9 @@ +{ + "utc": "2026-11-06T07:09:13.514447+00:00", + "trace_id": "ae34626f1564506e278461f8e8e32fa6", + "attachment_id": "f000feb7-d0a9-4338-b7d5-0371343cf6d1", + "status": "ready", + "elapsed_seconds": 61.422, + "db_attempt": 2, + "restart_count": 1 +} diff --git a/evidence/review-retry-summary.json b/evidence/review-retry-summary.json new file mode 100644 index 0000000..1a69600 --- /dev/null +++ b/evidence/review-retry-summary.json @@ -0,0 +1,27 @@ +{ + "trace_id": "ae34626f1564506e278461f8e8e32fa6", + "span_count": 21, + "span_names": { + "POST /api/v1/chats/{chat_id}/attachments": 1, + "postgres.user_for_token": 1, + "postgres.get_chat_for_member": 1, + "postgres.attachment_transaction": 3, + "s3.put_object": 4, + "postgres.create_attachment": 1, + "attachment.publish": 2, + "attachment.consume": 2, + "postgres.claim_attachment_job": 2, + "postgres.complete_attachment_job": 1, + "s3.delete_object": 1, + "s3.copy_object": 1, + "s3.get_object": 1 + }, + "producer_count": 2, + "consumer_count": 2, + "log_services": [ + "messenger-api", + "messenger-worker" + ], + "parent_relationships": "PASS", + "note": "Abrupt os._exit may lose unfinished spans of the crashed attempt; redelivery and committed retry remain visible." +} diff --git a/evidence/review-retry-trace.json b/evidence/review-retry-trace.json new file mode 100644 index 0000000..7df38cc --- /dev/null +++ b/evidence/review-retry-trace.json @@ -0,0 +1,870 @@ +{ + "captured_at": "2026-11-06T07:09:24.120944+00:00", + "query": "{service_name=~\"messenger-api|messenger-worker\"} | trace_id=\"ae34626f1564506e278461f8e8e32fa6\"", + "trace_id": "ae34626f1564506e278461f8e8e32fa6", + "logs": { + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "code_file_path": "/app/src/worker.py", + "code_function_name": "handle", + "code_line_number": "242", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1793948953512429924", + "outcome": "success", + "scope_name": "messenger.telemetry", + "service_instance_id": "8ab056f473cf", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "5b57d22d2d3eabde", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "ae34626f1564506e278461f8e8e32fa6" + }, + "values": [ + [ + "1793948953512353792", + "job.processed" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/worker.py", + "code_function_name": "handle", + "code_line_number": "242", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "observed_timestamp": "1793948893930110257", + "outcome": "skipped", + "scope_name": "messenger.telemetry", + "service_instance_id": "8ab056f473cf", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "632c5bb6629fa77c", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "ae34626f1564506e278461f8e8e32fa6" + }, + "values": [ + [ + "1793948893930080512", + "job.processed" + ] + ] + }, + { + "stream": { + "code_file_path": "/app/src/main.py", + "code_function_name": "add_request_id", + "code_line_number": "108", + "deployment_environment_name": "lab", + "detected_level": "info", + "flags": "1", + "method": "POST", + "observed_timestamp": "1793948892089943715", + "request_id": "fd45bf79-92fd-40d9-b1b0-2faeaa9abd35", + "route": "/api/v1/chats/{chat_id}/attachments", + "scope_name": "messenger.telemetry", + "service_instance_id": "d81222953eb2", + "service_name": "messenger-api", + "service_version": "1.6.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "8b7074677c4447e0", + "status": "202", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0", + "trace_id": "ae34626f1564506e278461f8e8e32fa6" + }, + "values": [ + [ + "1793948892089921536", + "http.request" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 50653466, + "linesProcessedPerSecond": 309238, + "totalBytesProcessed": 662083, + "totalLinesProcessed": 4042, + "execTime": 0.013071, + "queueTime": 0.000471, + "subqueries": 0, + "totalEntriesReturned": 3, + "splits": 2, + "shards": 2, + "totalPostFilterLines": 3, + "totalStructuredMetadataBytesProcessed": 613544 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 2, + "totalChunksMatched": 11, + "totalBatches": 3, + "totalLinesSent": 3, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": true, + "chunk": { + "headChunkBytes": 662083, + "headChunkLines": 4042, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 3, + "headChunkStructuredMetadataBytes": 613544, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 233000, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false + } + } + } + }, + "trace": { + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "d81222953eb2" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "i3B0Z3xER+A=", + "parentSpanId": "cPPbxzsVklc=", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1793948892082137923", + "endTimeUnixNano": "1793948892089988006", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "5ln8XViN22I=", + "parentSpanId": "i3B0Z3xER+A=", + "name": "postgres.user_for_token", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948892082560215", + "endTimeUnixNano": "1793948892083065215", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "ENe80xA2A2k=", + "parentSpanId": "i3B0Z3xER+A=", + "name": "postgres.get_chat_for_member", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948892083141590", + "endTimeUnixNano": "1793948892083547006", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "Z4dP2xX/VNg=", + "parentSpanId": "i3B0Z3xER+A=", + "name": "postgres.attachment_transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948892084939631", + "endTimeUnixNano": "1793948892089742131", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "zohqiMpi+QU=", + "parentSpanId": "Z4dP2xX/VNg=", + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948892085314173", + "endTimeUnixNano": "1793948892088217590", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "TjCZy2qExRI=", + "parentSpanId": "Z4dP2xX/VNg=", + "name": "postgres.create_attachment", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948892088244965", + "endTimeUnixNano": "1793948892089214631", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "b106c9e70b85" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-outbox" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "hHhcYiMN8r0=", + "parentSpanId": "TjCZy2qExRI=", + "name": "attachment.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1793948892280212882", + "endTimeUnixNano": "1793948892293371132", + "attributes": [ + { + "key": "messaging.retry.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "8ab056f473cf" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "YyxbtmKfp3w=", + "parentSpanId": "hHhcYiMN8r0=", + "name": "attachment.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1793948893852472632", + "endTimeUnixNano": "1793948893930183507", + "attributes": [ + { + "key": "messaging.retry.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "AoDGeW2ouFE=", + "parentSpanId": "YyxbtmKfp3w=", + "name": "postgres.attachment_transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948893915696007", + "endTimeUnixNano": "1793948893928589549", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "/+BHiGdE7Mg=", + "parentSpanId": "AoDGeW2ouFE=", + "name": "postgres.claim_attachment_job", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948893916050591", + "endTimeUnixNano": "1793948893928426049", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "b106c9e70b85" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-outbox" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "50KhQcpKHPM=", + "parentSpanId": "TjCZy2qExRI=", + "name": "attachment.publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1793948952742448132", + "endTimeUnixNano": "1793948952790699923", + "attributes": [ + { + "key": "messaging.retry.attempt", + "value": { + "intValue": "2" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "8ab056f473cf" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "W1fSLS0+q94=", + "parentSpanId": "50KhQcpKHPM=", + "flags": 768, + "name": "attachment.consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1793948952789545048", + "endTimeUnixNano": "1793948953512550965", + "attributes": [ + { + "key": "messaging.retry.attempt", + "value": { + "intValue": "2" + } + } + ], + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "FNIzX117Ifc=", + "parentSpanId": "W1fSLS0+q94=", + "flags": 256, + "name": "postgres.attachment_transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948952811862298", + "endTimeUnixNano": "1793948953510339924", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "/m/rSX5OYqA=", + "parentSpanId": "FNIzX117Ifc=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948953499890965", + "endTimeUnixNano": "1793948953502638090", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "N9dudhPIeg0=", + "parentSpanId": "FNIzX117Ifc=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948953502674424", + "endTimeUnixNano": "1793948953504062215", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "5WwABMSujUI=", + "parentSpanId": "FNIzX117Ifc=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948953504093882", + "endTimeUnixNano": "1793948953505309674", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "1EOfdNCSwms=", + "parentSpanId": "FNIzX117Ifc=", + "flags": 256, + "name": "postgres.complete_attachment_job", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948953505410007", + "endTimeUnixNano": "1793948953507989299", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "mjqJgDBztVQ=", + "parentSpanId": "FNIzX117Ifc=", + "flags": 256, + "name": "s3.delete_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948953508583424", + "endTimeUnixNano": "1793948953510069090", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "1.6.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "8ab056f473cf" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "lsvA6t1vfzY=", + "parentSpanId": "FNIzX117Ifc=", + "name": "postgres.claim_attachment_job", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948952812370090", + "endTimeUnixNano": "1793948952826915048", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "otmRM1OBdj8=", + "parentSpanId": "FNIzX117Ifc=", + "name": "s3.copy_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948952827010590", + "endTimeUnixNano": "1793948952861476507", + "status": {} + }, + { + "traceId": "rjRibxVkUG4nhGH46OMvpg==", + "spanId": "DvSIq363rzM=", + "parentSpanId": "FNIzX117Ifc=", + "name": "s3.get_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1793948952861520423", + "endTimeUnixNano": "1793948952863334340", + "status": {} + } + ] + } + ] + } + ] + } +} diff --git a/evidence/review-tempo-load.json b/evidence/review-tempo-load.json new file mode 100644 index 0000000..231ce6c --- /dev/null +++ b/evidence/review-tempo-load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 75.0, + "elapsed_seconds": 75.006, + "concurrency": 4, + "successful_requests": 40628, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 541.661, + "successful_latency_ms": { + "p50": 4.071, + "p95": 31.805, + "p99": 54.606 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/review-tempo-outage.json b/evidence/review-tempo-outage.json new file mode 100644 index 0000000..ec56245 --- /dev/null +++ b/evidence/review-tempo-outage.json @@ -0,0 +1,245 @@ +{ + "queries": { + "otelcol_exporter_enqueue_failed_spans_total": "sum(otelcol_exporter_enqueue_failed_spans_total{exporter=\"otlp/tempo\"}) or vector(0)", + "otelcol_exporter_send_failed_spans_total": "sum(otelcol_exporter_send_failed_spans_total{exporter=\"otlp/tempo\"}) or vector(0)", + "otelcol_exporter_sent_spans_total": "sum(otelcol_exporter_sent_spans_total{exporter=\"otlp/tempo\"}) or vector(0)", + "otelcol_exporter_queue_size": "sum(otelcol_exporter_queue_size{exporter=\"otlp/tempo\"}) or vector(0)", + "otelcol_exporter_queue_capacity": "sum(otelcol_exporter_queue_capacity{exporter=\"otlp/tempo\"}) or vector(0)" + }, + "samples": [ + { + "phase": "before", + "utc": "2026-11-06T07:09:41.743885+00:00", + "monotonic": 52353.585200125, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352596.0, + "otelcol_exporter_queue_size": 0.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "114.6MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:09:46.970039+00:00", + "monotonic": 52358.811228875, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 0.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "116.5MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:09:58.085092+00:00", + "monotonic": 52369.9262515, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 18.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "115.3MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:10:09.529528+00:00", + "monotonic": 52381.370598916, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 60.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "112.3MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:10:20.835788+00:00", + "monotonic": 52392.676654541, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 93.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "118.4MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:10:32.040068+00:00", + "monotonic": 52403.880798541, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 56361.0, + "otelcol_exporter_send_failed_spans_total": 639.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 125.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "120.5MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:10:43.165910+00:00", + "monotonic": 52415.006615083, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 71721.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "123MiB / 256MiB" + }, + { + "phase": "outage", + "utc": "2026-11-06T07:10:54.463041+00:00", + "monotonic": 52426.30365825, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 96297.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "125.9MiB / 256MiB" + }, + { + "phase": "before_restore", + "utc": "2026-11-06T07:11:05.578788+00:00", + "monotonic": 52437.419219458, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113193.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "126.9MiB / 256MiB" + }, + { + "phase": "recovery", + "utc": "2026-11-06T07:11:12.171271+00:00", + "monotonic": 52444.011731041, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113509.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "127MiB / 256MiB" + }, + { + "phase": "recovery", + "utc": "2026-11-06T07:11:18.348063+00:00", + "monotonic": 52450.188453375, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113557.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "126.8MiB / 256MiB" + }, + { + "phase": "recovery", + "utc": "2026-11-06T07:11:24.531975+00:00", + "monotonic": 52456.372292375, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113584.0, + "otelcol_exporter_send_failed_spans_total": 669.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "127.3MiB / 256MiB" + }, + { + "phase": "recovery", + "utc": "2026-11-06T07:11:30.708673+00:00", + "monotonic": 52462.548979291, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113593.0, + "otelcol_exporter_send_failed_spans_total": 762.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "127.1MiB / 256MiB" + }, + { + "phase": "recovery", + "utc": "2026-11-06T07:11:36.891275+00:00", + "monotonic": 52468.731487041, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113620.0, + "otelcol_exporter_send_failed_spans_total": 762.0, + "otelcol_exporter_sent_spans_total": 352617.0, + "otelcol_exporter_queue_size": 128.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "128MiB / 256MiB" + }, + { + "phase": "recovery_followup", + "utc": "2026-11-06T07:12:45.670764+00:00", + "monotonic": 52537.510424666, + "metrics": { + "otelcol_exporter_enqueue_failed_spans_total": 113629.0, + "otelcol_exporter_send_failed_spans_total": 1411.0, + "otelcol_exporter_sent_spans_total": 417456.0, + "otelcol_exporter_queue_size": 0.0, + "otelcol_exporter_queue_capacity": 128.0 + }, + "container_id": "022ca1971dd88b3dcfdd23ec5c4951830655070549631c29a2bddcedf354f45c", + "memory": "128.8MiB / 256MiB" + } + ], + "counter_deltas": { + "otelcol_exporter_enqueue_failed_spans_total": 57268.0, + "otelcol_exporter_send_failed_spans_total": 772.0, + "otelcol_exporter_sent_spans_total": 64860.0 + }, + "load": { + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 75.0, + "elapsed_seconds": 75.006, + "concurrency": 4, + "successful_requests": 40628, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 541.661, + "successful_latency_ms": { + "p50": 4.071, + "p95": 31.805, + "p99": 54.606 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "result": "PASS", + "recovery_verified": true +} diff --git a/evidence/review-tempo-persistence.json b/evidence/review-tempo-persistence.json new file mode 100644 index 0000000..7a65c4d --- /dev/null +++ b/evidence/review-tempo-persistence.json @@ -0,0 +1,93 @@ +{ + "utc": "2026-11-06T07:15:48.364407+00:00", + "trace_id": "ae34626f1564506e278461f8e8e32fa6", + "before_artifact": "review-retry-trace.json", + "query": "Tempo /api/traces/ae34626f1564506e278461f8e8e32fa6", + "after_spans": [ + [ + "/+BHiGdE7Mg=", + "postgres.claim_attachment_job" + ], + [ + "/m/rSX5OYqA=", + "s3.put_object" + ], + [ + "1EOfdNCSwms=", + "postgres.complete_attachment_job" + ], + [ + "50KhQcpKHPM=", + "attachment.publish" + ], + [ + "5WwABMSujUI=", + "s3.put_object" + ], + [ + "5ln8XViN22I=", + "postgres.user_for_token" + ], + [ + "AoDGeW2ouFE=", + "postgres.attachment_transaction" + ], + [ + "DvSIq363rzM=", + "s3.get_object" + ], + [ + "ENe80xA2A2k=", + "postgres.get_chat_for_member" + ], + [ + "FNIzX117Ifc=", + "postgres.attachment_transaction" + ], + [ + "N9dudhPIeg0=", + "s3.put_object" + ], + [ + "TjCZy2qExRI=", + "postgres.create_attachment" + ], + [ + "W1fSLS0+q94=", + "attachment.consume" + ], + [ + "YyxbtmKfp3w=", + "attachment.consume" + ], + [ + "Z4dP2xX/VNg=", + "postgres.attachment_transaction" + ], + [ + "hHhcYiMN8r0=", + "attachment.publish" + ], + [ + "i3B0Z3xER+A=", + "POST /api/v1/chats/{chat_id}/attachments" + ], + [ + "lsvA6t1vfzY=", + "postgres.claim_attachment_job" + ], + [ + "mjqJgDBztVQ=", + "s3.delete_object" + ], + [ + "otmRM1OBdj8=", + "s3.copy_object" + ], + [ + "zohqiMpi+QU=", + "s3.put_object" + ] + ], + "same_spans_after_restart": true +} diff --git a/evidence/slo-analysis.json b/evidence/slo-analysis.json index 9c5d11c..394d077 100644 --- a/evidence/slo-analysis.json +++ b/evidence/slo-analysis.json @@ -3,48 +3,54 @@ "monitor_errors": 0, "phases": { "burst": { - "observed_lock_seconds": 75.033, + "observed_lock_seconds": 75.001, "alerts": { "MessengerApiBurnFast": { "active_before_lock": false, - "first_firing": "2026-11-05T19:30:49.560423+00:00", + "first_firing": "2026-11-06T06:54:08.779613+00:00", "first_firing_after_incident": false, - "detection_seconds": 55.088, - "first_clear": "2026-11-05T19:31:24.687321+00:00", - "recovery_seconds": 15.181 + "detection_seconds": 54.676, + "first_clear": "2026-11-06T06:54:43.864649+00:00", + "recovery_seconds": 14.76 }, "MessengerApiBurnSlow": { "active_before_lock": false, - "first_firing": "2026-11-05T19:31:14.651231+00:00", - "first_firing_after_incident": true, - "detection_seconds": 80.179, - "first_clear": "2026-11-05T19:31:24.687321+00:00", - "recovery_seconds": 15.181 + "first_firing": null, + "first_firing_after_incident": false, + "detection_seconds": null, + "first_clear": null, + "recovery_seconds": null } } }, "sustained": { - "observed_lock_seconds": 180.134, + "observed_lock_seconds": 180.097, "alerts": { "MessengerApiBurnFast": { "active_before_lock": false, - "first_firing": "2026-11-05T19:33:40.096081+00:00", + "first_firing": "2026-11-06T06:56:59.279143+00:00", "first_firing_after_incident": false, - "detection_seconds": 50.278, - "first_clear": "2026-11-05T19:36:10.387240+00:00", - "recovery_seconds": 10.648 + "detection_seconds": 49.86, + "first_clear": "2026-11-06T06:59:24.818695+00:00", + "recovery_seconds": 15.301 }, "MessengerApiBurnSlow": { "active_before_lock": false, - "first_firing": "2026-11-05T19:34:20.245337+00:00", + "first_firing": "2026-11-06T06:57:39.433399+00:00", "first_firing_after_incident": false, - "detection_seconds": 90.428, - "first_clear": "2026-11-05T19:37:53.534744+00:00", - "recovery_seconds": 30.724 + "detection_seconds": 90.013, + "first_clear": "2026-11-06T06:59:49.911576+00:00", + "recovery_seconds": 40.394 } } } }, + "timing_quality": { + "max_wall_monotonic_disagreement_seconds": 0.0, + "max_observed_sampling_gap_seconds": 5.037, + "valid_for_elapsed_time_claims": true, + "note": "Disagreement does not identify its cause; rerun if invalid. Precision is bounded by actual gaps, not the configured sleep." + }, "baseline": { "samples": 13, "samples_with_burn_alert": 0, diff --git a/evidence/slo-incidents.json b/evidence/slo-incidents.json index 742ed8f..890db31 100644 --- a/evidence/slo-incidents.json +++ b/evidence/slo-incidents.json @@ -3,267 +3,296 @@ "periods": { "burst": { "lock_observed": { - "time": "2026-11-05T19:29:54.472744+00:00", - "monotonic": 43527.079238041, + "time": "2026-11-06T06:53:14.103079+00:00", + "monotonic": 51365.954057666, "command_started": { - "time": "2026-11-05T19:29:54.163828+00:00", - "monotonic": 43526.770364958 + "time": "2026-11-06T06:53:13.417102+00:00", + "monotonic": 51365.268133416 } }, "command_completed": { - "time": "2026-11-05T19:31:09.505932+00:00", - "monotonic": 43602.112587416 + "time": "2026-11-06T06:54:29.104850+00:00", + "monotonic": 51440.955130083 } }, "sustained": { "lock_observed": { - "time": "2026-11-05T19:32:49.817996+00:00", - "monotonic": 43702.424801208, + "time": "2026-11-06T06:56:09.419160+00:00", + "monotonic": 51541.268408583, "command_started": { - "time": "2026-11-05T19:32:49.516057+00:00", - "monotonic": 43702.122908708 + "time": "2026-11-06T06:56:09.111306+00:00", + "monotonic": 51540.960600125 } }, "command_completed": { - "time": "2026-11-05T19:35:59.744503+00:00", - "monotonic": 43882.558427291 + "time": "2026-11-06T06:59:09.517513+00:00", + "monotonic": 51721.365017541 } } }, "samples": [ { - "time": "2026-11-05T19:28:54.176617+00:00", - "monotonic": 43466.783021166, + "time": "2026-11-06T06:52:13.424045+00:00", + "monotonic": 51305.275633833, "phase": "baseline", "counts": { "ok": 1, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:28:59.184768+00:00", - "monotonic": 43471.791177208, + "time": "2026-11-06T06:52:18.436944+00:00", + "monotonic": 51310.2884845, "phase": "baseline", "counts": { "ok": 15, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:04.205356+00:00", - "monotonic": 43476.8117975, + "time": "2026-11-06T06:52:23.451635+00:00", + "monotonic": 51315.303133083, "phase": "baseline", "counts": { "ok": 30, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:09.226269+00:00", - "monotonic": 43481.832718708, + "time": "2026-11-06T06:52:28.466169+00:00", + "monotonic": 51320.317629083, "phase": "baseline", "counts": { "ok": 45, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:14.243046+00:00", - "monotonic": 43486.849499708, + "time": "2026-11-06T06:52:33.473068+00:00", + "monotonic": 51325.324459541, "phase": "baseline", "counts": { "ok": 60, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:19.249330+00:00", - "monotonic": 43491.855772208, + "time": "2026-11-06T06:52:38.485045+00:00", + "monotonic": 51330.336387, "phase": "baseline", "counts": { "ok": 75, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:24.265281+00:00", - "monotonic": 43496.871750375, + "time": "2026-11-06T06:52:43.495166+00:00", + "monotonic": 51335.346459208, "phase": "baseline", "counts": { "ok": 90, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:29.284910+00:00", - "monotonic": 43501.891384583, + "time": "2026-11-06T06:52:48.528605+00:00", + "monotonic": 51340.379879458, "phase": "baseline", "counts": { "ok": 105, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:34.295325+00:00", - "monotonic": 43506.901793458, + "time": "2026-11-06T06:52:53.543457+00:00", + "monotonic": 51345.394690083, "phase": "baseline", "counts": { "ok": 120, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:39.306441+00:00", - "monotonic": 43511.91291825, + "time": "2026-11-06T06:52:58.563039+00:00", + "monotonic": 51350.414196125, "phase": "baseline", "counts": { "ok": 135, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:44.313768+00:00", - "monotonic": 43516.920254333, + "time": "2026-11-06T06:53:03.585925+00:00", + "monotonic": 51355.437051291, "phase": "baseline", "counts": { "ok": 150, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:49.331915+00:00", - "monotonic": 43521.938423708, + "time": "2026-11-06T06:53:08.603589+00:00", + "monotonic": 51360.454639416, "phase": "baseline", "counts": { "ok": 165, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:54.351411+00:00", - "monotonic": 43526.957913708, + "time": "2026-11-06T06:53:13.615700+00:00", + "monotonic": 51365.466690666, "phase": "baseline", "counts": { "ok": 180, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:29:59.372201+00:00", - "monotonic": 43531.978732458, + "time": "2026-11-06T06:53:18.628657+00:00", + "monotonic": 51370.479612125, "phase": "burst", "counts": { "ok": 180, "failed": 0 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:04.386019+00:00", - "monotonic": 43536.992552291, + "time": "2026-11-06T06:53:23.650959+00:00", + "monotonic": 51375.501877458, "phase": "burst", "counts": { "ok": 180, "failed": 3 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:09.401930+00:00", - "monotonic": 43542.0084625, + "time": "2026-11-06T06:53:28.671969+00:00", + "monotonic": 51380.522831125, "phase": "burst", "counts": { "ok": 180, "failed": 6 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:14.419029+00:00", - "monotonic": 43547.025585291, + "time": "2026-11-06T06:53:33.683094+00:00", + "monotonic": 51385.533890375, "phase": "burst", "counts": { "ok": 180, "failed": 9 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:19.433563+00:00", - "monotonic": 43552.040130916, + "time": "2026-11-06T06:53:38.695580+00:00", + "monotonic": 51390.546326541, "phase": "burst", "counts": { "ok": 180, "failed": 12 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:24.450146+00:00", - "monotonic": 43557.05672225, + "time": "2026-11-06T06:53:43.708619+00:00", + "monotonic": 51395.559316583, "phase": "burst", "counts": { "ok": 180, "failed": 15 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:29.473233+00:00", - "monotonic": 43562.079822583, + "time": "2026-11-06T06:53:48.719046+00:00", + "monotonic": 51400.569692583, "phase": "burst", "counts": { "ok": 180, "failed": 15 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:30:34.493955+00:00", - "monotonic": 43567.1005505, + "time": "2026-11-06T06:53:53.733123+00:00", + "monotonic": 51405.583727375, "phase": "burst", "counts": { "ok": 180, "failed": 18 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:30:39.519370+00:00", - "monotonic": 43572.125976166, + "time": "2026-11-06T06:53:58.750268+00:00", + "monotonic": 51410.600817208, "phase": "burst", "counts": { "ok": 180, "failed": 21 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:30:44.535535+00:00", - "monotonic": 43577.142128416, + "time": "2026-11-06T06:54:03.765059+00:00", + "monotonic": 51415.615563416, "phase": "burst", "counts": { "ok": 180, "failed": 24 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:30:49.560423+00:00", - "monotonic": 43582.167042875, + "time": "2026-11-06T06:54:08.779613+00:00", + "monotonic": 51420.630081875, "phase": "burst", "counts": { "ok": 180, @@ -271,11 +300,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:30:54.583572+00:00", - "monotonic": 43587.190215125, + "time": "2026-11-06T06:54:13.798324+00:00", + "monotonic": 51425.648740833, "phase": "burst", "counts": { "ok": 180, @@ -283,11 +316,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:30:59.600250+00:00", - "monotonic": 43592.206889708, + "time": "2026-11-06T06:54:18.814338+00:00", + "monotonic": 51430.664704291, "phase": "burst", "counts": { "ok": 180, @@ -295,11 +332,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:31:04.616531+00:00", - "monotonic": 43597.223167708, + "time": "2026-11-06T06:54:23.820189+00:00", + "monotonic": 51435.670496541, "phase": "burst", "counts": { "ok": 180, @@ -307,319 +348,378 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:31:09.632643+00:00", - "monotonic": 43602.239279708, - "phase": "burst_recovery", + "time": "2026-11-06T06:54:28.827381+00:00", + "monotonic": 51440.677629875, + "phase": "burst", "counts": { - "ok": 183, + "ok": 180, "failed": 36 }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:31:14.651231+00:00", - "monotonic": 43607.257891625, + "time": "2026-11-06T06:54:33.839856+00:00", + "monotonic": 51445.690054291, "phase": "burst_recovery", "counts": { - "ok": 198, + "ok": 195, "failed": 36 }, "firing": [ + "MessengerApiBurnFast" + ], + "all_firing": [ "MessengerApiBurnFast", - "MessengerApiBurnSlow" + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:31:19.663273+00:00", - "monotonic": 43612.269926416, + "time": "2026-11-06T06:54:38.847760+00:00", + "monotonic": 51450.697911083, "phase": "burst_recovery", "counts": { - "ok": 211, + "ok": 210, "failed": 36 }, "firing": [ + "MessengerApiBurnFast" + ], + "all_firing": [ "MessengerApiBurnFast", - "MessengerApiBurnSlow" + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:31:24.687321+00:00", - "monotonic": 43617.294002291, + "time": "2026-11-06T06:54:43.864649+00:00", + "monotonic": 51455.714782791, "phase": "burst_recovery", "counts": { "ok": 225, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:29.708468+00:00", - "monotonic": 43622.315158041, + "time": "2026-11-06T06:54:48.888857+00:00", + "monotonic": 51460.738907041, "phase": "burst_recovery", "counts": { "ok": 240, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:34.728166+00:00", - "monotonic": 43627.334856708, + "time": "2026-11-06T06:54:53.901671+00:00", + "monotonic": 51465.751684583, "phase": "burst_recovery", "counts": { "ok": 255, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:39.733767+00:00", - "monotonic": 43632.340459958, + "time": "2026-11-06T06:54:58.907581+00:00", + "monotonic": 51470.75753375, "phase": "burst_recovery", "counts": { "ok": 270, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:44.755480+00:00", - "monotonic": 43637.36218675, + "time": "2026-11-06T06:55:03.928237+00:00", + "monotonic": 51475.778157458, "phase": "burst_recovery", "counts": { "ok": 285, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:49.769677+00:00", - "monotonic": 43642.376390333, + "time": "2026-11-06T06:55:08.948107+00:00", + "monotonic": 51480.797979708, "phase": "burst_recovery", "counts": { "ok": 300, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:54.778133+00:00", - "monotonic": 43647.384860875, + "time": "2026-11-06T06:55:13.965736+00:00", + "monotonic": 51485.815554833, "phase": "burst_recovery", "counts": { "ok": 315, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:31:59.788234+00:00", - "monotonic": 43652.394963833, + "time": "2026-11-06T06:55:18.983097+00:00", + "monotonic": 51490.83286575, "phase": "burst_recovery", "counts": { "ok": 330, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:04.797259+00:00", - "monotonic": 43657.403998, + "time": "2026-11-06T06:55:23.996671+00:00", + "monotonic": 51495.846385125, "phase": "burst_recovery", "counts": { "ok": 345, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:09.813581+00:00", - "monotonic": 43662.42033325, + "time": "2026-11-06T06:55:29.010045+00:00", + "monotonic": 51500.859710625, "phase": "burst_recovery", "counts": { - "ok": 360, + "ok": 357, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:14.823961+00:00", - "monotonic": 43667.430715458, + "time": "2026-11-06T06:55:34.032445+00:00", + "monotonic": 51505.882069958, "phase": "burst_recovery", "counts": { - "ok": 375, + "ok": 372, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:19.834831+00:00", - "monotonic": 43672.441596833, + "time": "2026-11-06T06:55:39.048403+00:00", + "monotonic": 51510.897980041, "phase": "burst_recovery", "counts": { - "ok": 390, + "ok": 387, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:24.847866+00:00", - "monotonic": 43677.454636958, + "time": "2026-11-06T06:55:44.065766+00:00", + "monotonic": 51515.915294375, "phase": "burst_recovery", "counts": { - "ok": 405, + "ok": 402, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:29.862747+00:00", - "monotonic": 43682.469528375, + "time": "2026-11-06T06:55:49.081176+00:00", + "monotonic": 51520.930651458, "phase": "burst_recovery", "counts": { "ok": 417, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:34.870564+00:00", - "monotonic": 43687.47735275, + "time": "2026-11-06T06:55:54.097495+00:00", + "monotonic": 51525.946920083, "phase": "burst_recovery", "counts": { "ok": 432, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:39.880308+00:00", - "monotonic": 43692.487106, + "time": "2026-11-06T06:55:59.116926+00:00", + "monotonic": 51530.966305541, "phase": "burst_recovery", "counts": { "ok": 447, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:44.899942+00:00", - "monotonic": 43697.506751, + "time": "2026-11-06T06:56:04.123010+00:00", + "monotonic": 51535.972316291, "phase": "burst_recovery", "counts": { "ok": 462, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:49.916853+00:00", - "monotonic": 43702.523669458, + "time": "2026-11-06T06:56:09.139780+00:00", + "monotonic": 51540.989044958, + "phase": "burst_recovery", + "counts": { + "ok": 477, + "failed": 36 + }, + "firing": [], + "all_firing": [] + }, + { + "time": "2026-11-06T06:56:14.158912+00:00", + "monotonic": 51546.008146666, "phase": "sustained", "counts": { "ok": 477, "failed": 36 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:32:54.942437+00:00", - "monotonic": 43707.549288958, - "phase": "sustained", - "counts": { - "ok": 477, - "failed": 36 - }, - "firing": [] - }, - { - "time": "2026-11-05T19:32:59.957394+00:00", - "monotonic": 43712.564228083, + "time": "2026-11-06T06:56:19.170927+00:00", + "monotonic": 51551.020093583, "phase": "sustained", "counts": { "ok": 477, "failed": 39 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:04.970687+00:00", - "monotonic": 43717.577532333, + "time": "2026-11-06T06:56:24.181412+00:00", + "monotonic": 51556.030526708, "phase": "sustained", "counts": { "ok": 477, "failed": 42 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:09.993536+00:00", - "monotonic": 43722.600420875, + "time": "2026-11-06T06:56:29.189049+00:00", + "monotonic": 51561.038109333, "phase": "sustained", "counts": { "ok": 477, "failed": 45 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:15.009372+00:00", - "monotonic": 43727.616250583, + "time": "2026-11-06T06:56:34.205264+00:00", + "monotonic": 51566.054276125, "phase": "sustained", "counts": { "ok": 477, "failed": 48 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:20.031828+00:00", - "monotonic": 43732.638718, + "time": "2026-11-06T06:56:39.211225+00:00", + "monotonic": 51571.060184458, "phase": "sustained", "counts": { "ok": 477, "failed": 51 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:25.051997+00:00", - "monotonic": 43737.658880125, + "time": "2026-11-06T06:56:44.222128+00:00", + "monotonic": 51576.07104075, "phase": "sustained", "counts": { "ok": 477, "failed": 51 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:33:30.067799+00:00", - "monotonic": 43742.674685791, + "time": "2026-11-06T06:56:49.245180+00:00", + "monotonic": 51581.094060708, "phase": "sustained", "counts": { "ok": 477, "failed": 54 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:33:35.075453+00:00", - "monotonic": 43747.682349166, + "time": "2026-11-06T06:56:54.262383+00:00", + "monotonic": 51586.11121325, "phase": "sustained", "counts": { "ok": 477, "failed": 57 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:33:40.096081+00:00", - "monotonic": 43752.703001416, + "time": "2026-11-06T06:56:59.279143+00:00", + "monotonic": 51591.127925625, "phase": "sustained", "counts": { "ok": 477, @@ -627,11 +727,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:33:45.119089+00:00", - "monotonic": 43757.7260235, + "time": "2026-11-06T06:57:04.297663+00:00", + "monotonic": 51596.146368833, "phase": "sustained", "counts": { "ok": 477, @@ -639,11 +743,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:33:50.133709+00:00", - "monotonic": 43762.740644916, + "time": "2026-11-06T06:57:09.310113+00:00", + "monotonic": 51601.158780166, "phase": "sustained", "counts": { "ok": 477, @@ -651,11 +759,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:33:55.161233+00:00", - "monotonic": 43767.768187291, + "time": "2026-11-06T06:57:14.323594+00:00", + "monotonic": 51606.172214833, "phase": "sustained", "counts": { "ok": 477, @@ -663,11 +775,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:00.176785+00:00", - "monotonic": 43772.783745208, + "time": "2026-11-06T06:57:19.344779+00:00", + "monotonic": 51611.193360083, "phase": "sustained", "counts": { "ok": 477, @@ -675,11 +791,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:05.196858+00:00", - "monotonic": 43777.803812291, + "time": "2026-11-06T06:57:24.364531+00:00", + "monotonic": 51616.213063416, "phase": "sustained", "counts": { "ok": 477, @@ -687,11 +807,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:10.212935+00:00", - "monotonic": 43782.819901041, + "time": "2026-11-06T06:57:29.393320+00:00", + "monotonic": 51621.241828666, "phase": "sustained", "counts": { "ok": 477, @@ -699,11 +823,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:15.231470+00:00", - "monotonic": 43787.838454541, + "time": "2026-11-06T06:57:34.412151+00:00", + "monotonic": 51626.260583541, "phase": "sustained", "counts": { "ok": 477, @@ -711,11 +839,15 @@ }, "firing": [ "MessengerApiBurnFast" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:20.245337+00:00", - "monotonic": 43792.852329875, + "time": "2026-11-06T06:57:39.433399+00:00", + "monotonic": 51631.281782041, "phase": "sustained", "counts": { "ok": 477, @@ -724,11 +856,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:25.257984+00:00", - "monotonic": 43797.864980958, + "time": "2026-11-06T06:57:44.449034+00:00", + "monotonic": 51636.29735525, "phase": "sustained", "counts": { "ok": 477, @@ -737,11 +874,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:30.282024+00:00", - "monotonic": 43802.889036875, + "time": "2026-11-06T06:57:49.461390+00:00", + "monotonic": 51641.309667083, "phase": "sustained", "counts": { "ok": 477, @@ -750,11 +892,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:35.303328+00:00", - "monotonic": 43807.910348833, + "time": "2026-11-06T06:57:54.483943+00:00", + "monotonic": 51646.332173708, "phase": "sustained", "counts": { "ok": 477, @@ -763,11 +910,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:40.328342+00:00", - "monotonic": 43812.935370625, + "time": "2026-11-06T06:57:59.509458+00:00", + "monotonic": 51651.357645291, "phase": "sustained", "counts": { "ok": 477, @@ -776,11 +928,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:45.343604+00:00", - "monotonic": 43817.950633291, + "time": "2026-11-06T06:58:04.535029+00:00", + "monotonic": 51656.383169208, "phase": "sustained", "counts": { "ok": 477, @@ -789,11 +946,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:50.359471+00:00", - "monotonic": 43822.966518166, + "time": "2026-11-06T06:58:09.551012+00:00", + "monotonic": 51661.399097541, "phase": "sustained", "counts": { "ok": 477, @@ -802,11 +964,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:34:55.367578+00:00", - "monotonic": 43827.974616416, + "time": "2026-11-06T06:58:14.567487+00:00", + "monotonic": 51666.415528875, "phase": "sustained", "counts": { "ok": 477, @@ -815,11 +982,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:00.386215+00:00", - "monotonic": 43832.993284208, + "time": "2026-11-06T06:58:19.584767+00:00", + "monotonic": 51671.432743666, "phase": "sustained", "counts": { "ok": 477, @@ -828,11 +1000,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:05.400579+00:00", - "monotonic": 43838.0076475, + "time": "2026-11-06T06:58:24.610726+00:00", + "monotonic": 51676.458661375, "phase": "sustained", "counts": { "ok": 477, @@ -841,11 +1018,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:10.422157+00:00", - "monotonic": 43843.0292365, + "time": "2026-11-06T06:58:29.636000+00:00", + "monotonic": 51681.483882625, "phase": "sustained", "counts": { "ok": 477, @@ -854,11 +1036,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:15.442075+00:00", - "monotonic": 43848.049173416, + "time": "2026-11-06T06:58:34.651395+00:00", + "monotonic": 51686.499233166, "phase": "sustained", "counts": { "ok": 477, @@ -867,11 +1054,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:20.452042+00:00", - "monotonic": 43853.05914025, + "time": "2026-11-06T06:58:39.665021+00:00", + "monotonic": 51691.512806666, "phase": "sustained", "counts": { "ok": 477, @@ -880,11 +1072,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:25.461182+00:00", - "monotonic": 43858.068280916, + "time": "2026-11-06T06:58:44.671694+00:00", + "monotonic": 51696.519412875, "phase": "sustained", "counts": { "ok": 477, @@ -893,11 +1090,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:30.480197+00:00", - "monotonic": 43863.087333625, + "time": "2026-11-06T06:58:49.683388+00:00", + "monotonic": 51701.531061666, "phase": "sustained", "counts": { "ok": 477, @@ -906,11 +1108,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:35.504075+00:00", - "monotonic": 43868.111188291, + "time": "2026-11-06T06:58:54.694272+00:00", + "monotonic": 51706.541890833, "phase": "sustained", "counts": { "ok": 477, @@ -919,11 +1126,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:40.521486+00:00", - "monotonic": 43873.129961791, + "time": "2026-11-06T06:58:59.718693+00:00", + "monotonic": 51711.56628275, "phase": "sustained", "counts": { "ok": 477, @@ -932,11 +1144,16 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:35:55.348285+00:00", - "monotonic": 43878.158763833, + "time": "2026-11-06T06:59:04.743549+00:00", + "monotonic": 51716.59108925, "phase": "sustained", "counts": { "ok": 477, @@ -945,226 +1162,289 @@ "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:36:00.359177+00:00", - "monotonic": 43883.173501041, + "time": "2026-11-06T06:59:09.765218+00:00", + "monotonic": 51721.61271225, "phase": "sustained_recovery", "counts": { - "ok": 480, - "failed": 123 + "ok": 477, + "failed": 126 }, "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:36:05.367876+00:00", - "monotonic": 43888.184996, + "time": "2026-11-06T06:59:14.783393+00:00", + "monotonic": 51726.630834, "phase": "sustained_recovery", "counts": { - "ok": 495, - "failed": 123 + "ok": 492, + "failed": 126 }, "firing": [ "MessengerApiBurnFast", "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:36:10.387240+00:00", - "monotonic": 43893.206441208, + "time": "2026-11-06T06:59:19.797030+00:00", + "monotonic": 51731.644447416, "phase": "sustained_recovery", "counts": { - "ok": 510, - "failed": 123 + "ok": 507, + "failed": 126 + }, + "firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnFast", + "MessengerApiBurnSlow", + "MessengerHighErrorRate" + ] + }, + { + "time": "2026-11-06T06:59:24.818695+00:00", + "monotonic": 51736.666055666, + "phase": "sustained_recovery", + "counts": { + "ok": 522, + "failed": 126 }, "firing": [ "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:36:15.397936+00:00", - "monotonic": 43898.218563625, + "time": "2026-11-06T06:59:29.831731+00:00", + "monotonic": 51741.679002958, "phase": "sustained_recovery", "counts": { - "ok": 525, - "failed": 123 + "ok": 537, + "failed": 126 }, "firing": [ "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:36:20.421727+00:00", - "monotonic": 43903.243449875, + "time": "2026-11-06T06:59:34.843659+00:00", + "monotonic": 51746.690912916, "phase": "sustained_recovery", "counts": { - "ok": 540, - "failed": 123 + "ok": 551, + "failed": 126 }, "firing": [ "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:37:48.510428+00:00", - "monotonic": 43908.257572375, + "time": "2026-11-06T06:59:39.857744+00:00", + "monotonic": 51751.704937458, "phase": "sustained_recovery", "counts": { - "ok": 555, - "failed": 123 + "ok": 564, + "failed": 126 }, "firing": [ "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnSlow", + "MessengerHighErrorRate" ] }, { - "time": "2026-11-05T19:37:53.534744+00:00", - "monotonic": 43913.282484708, + "time": "2026-11-06T06:59:44.874968+00:00", + "monotonic": 51756.722113666, "phase": "sustained_recovery", "counts": { - "ok": 570, - "failed": 123 + "ok": 579, + "failed": 126 }, - "firing": [] + "firing": [ + "MessengerApiBurnSlow" + ], + "all_firing": [ + "MessengerApiBurnSlow", + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:37:58.549296+00:00", - "monotonic": 43918.29743975, + "time": "2026-11-06T06:59:49.911576+00:00", + "monotonic": 51761.75867025, "phase": "sustained_recovery", "counts": { - "ok": 585, - "failed": 123 + "ok": 594, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:38:03.567430+00:00", - "monotonic": 43923.315891, + "time": "2026-11-06T06:59:54.935006+00:00", + "monotonic": 51766.782047625, "phase": "sustained_recovery", "counts": { - "ok": 600, - "failed": 123 + "ok": 609, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:38:08.593041+00:00", - "monotonic": 43928.341731291, + "time": "2026-11-06T06:59:59.946639+00:00", + "monotonic": 51771.793639375, "phase": "sustained_recovery", "counts": { - "ok": 615, - "failed": 123 + "ok": 624, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [ + "MessengerHighErrorRate" + ] }, { - "time": "2026-11-05T19:38:13.607305+00:00", - "monotonic": 43933.356177, + "time": "2026-11-06T07:00:04.962978+00:00", + "monotonic": 51776.809903958, "phase": "sustained_recovery", "counts": { - "ok": 630, - "failed": 123 + "ok": 639, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:18.617266+00:00", - "monotonic": 43938.366246166, + "time": "2026-11-06T07:00:09.975033+00:00", + "monotonic": 51781.821923208, "phase": "sustained_recovery", "counts": { - "ok": 642, - "failed": 123 + "ok": 654, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:23.636630+00:00", - "monotonic": 43943.385724875, + "time": "2026-11-06T07:00:14.986604+00:00", + "monotonic": 51786.833433416, "phase": "sustained_recovery", "counts": { - "ok": 657, - "failed": 123 + "ok": 669, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:28.654169+00:00", - "monotonic": 43948.403359125, + "time": "2026-11-06T07:00:20.002499+00:00", + "monotonic": 51791.849318, "phase": "sustained_recovery", "counts": { - "ok": 672, - "failed": 123 + "ok": 684, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:33.670193+00:00", - "monotonic": 43953.419418, + "time": "2026-11-06T07:00:25.020127+00:00", + "monotonic": 51796.866889708, "phase": "sustained_recovery", "counts": { - "ok": 687, - "failed": 123 + "ok": 699, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:38.676034+00:00", - "monotonic": 43958.425318541, + "time": "2026-11-06T07:00:30.038727+00:00", + "monotonic": 51801.88541975, "phase": "sustained_recovery", "counts": { - "ok": 702, - "failed": 123 + "ok": 714, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:43.699279+00:00", - "monotonic": 43963.448615416, + "time": "2026-11-06T07:00:35.054844+00:00", + "monotonic": 51806.901469125, "phase": "sustained_recovery", "counts": { - "ok": 717, - "failed": 123 + "ok": 729, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:48.718747+00:00", - "monotonic": 43968.468122083, + "time": "2026-11-06T07:00:40.071309+00:00", + "monotonic": 51811.917906208, "phase": "sustained_recovery", "counts": { - "ok": 732, - "failed": 123 + "ok": 744, + "failed": 126 }, - "firing": [] + "firing": [], + "all_firing": [] }, { - "time": "2026-11-05T19:38:53.742289+00:00", - "monotonic": 43973.491697375, + "time": "2026-11-06T07:00:45.095471+00:00", + "monotonic": 51816.942013541, "phase": "sustained_recovery", "counts": { - "ok": 747, - "failed": 123 + "ok": 757, + "failed": 126 }, - "firing": [] - }, - { - "time": "2026-11-05T19:38:58.757203+00:00", - "monotonic": 43978.50662725, - "phase": "sustained_recovery", - "counts": { - "ok": 762, - "failed": 123 - }, - "firing": [] + "firing": [], + "all_firing": [] } ], - "end_time": "2026-11-05T19:39:02.820853+00:00", + "end_time": "2026-11-06T07:00:49.528862+00:00", "final_counts": { - "ok": 774, - "failed": 123 + "ok": 771, + "failed": 126 } } diff --git a/evidence/slo-ready-incident.json b/evidence/slo-ready-incident.json index 5eff503..a14cd2c 100644 --- a/evidence/slo-ready-incident.json +++ b/evidence/slo-ready-incident.json @@ -1,211 +1,302 @@ { "samples": [ { - "time": "2026-11-06T06:18:43.857552+00:00", - "monotonic": 49295.73000975, + "time": "2026-11-06T07:05:52.042304+00:00", + "monotonic": 52123.885794541, "phase": "baseline", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:18:49.195942+00:00", - "monotonic": 49301.068345333, + "time": "2026-11-06T07:05:57.341477+00:00", + "monotonic": 52129.184913625, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:18:54.211236+00:00", - "monotonic": 49306.083585125, + "time": "2026-11-06T07:06:02.378900+00:00", + "monotonic": 52134.222306833, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:18:59.238528+00:00", - "monotonic": 49311.110847333, + "time": "2026-11-06T07:06:07.432741+00:00", + "monotonic": 52139.276099166, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 1.943177, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:04.276851+00:00", - "monotonic": 49316.149107833, + "time": "2026-11-06T07:06:12.488438+00:00", + "monotonic": 52144.331749416, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 6.984557, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:09.316299+00:00", - "monotonic": 49321.18853025, + "time": "2026-11-06T07:06:17.540313+00:00", + "monotonic": 52149.383570875, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 12.016421, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:14.361014+00:00", - "monotonic": 49326.233190833, + "time": "2026-11-06T07:06:22.591885+00:00", + "monotonic": 52154.435094083, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 17.061549, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:19.399292+00:00", - "monotonic": 49331.271419, + "time": "2026-11-06T07:06:27.629967+00:00", + "monotonic": 52159.473106666, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 22.089736, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:24.439389+00:00", - "monotonic": 49336.31146675, + "time": "2026-11-06T07:06:32.659343+00:00", + "monotonic": 52164.502452166, "phase": "worker_stopped", "firing": [], - "eligible": 9.0, - "bad": 4.0 + "all_firing": [], + "queue_depth": 8.0, + "queue_age": 27.133281, + "eligible": 17.0, + "bad": 16.0 }, { - "time": "2026-11-06T06:19:29.469853+00:00", - "monotonic": 49341.341864875, + "time": "2026-11-06T07:06:37.707157+00:00", + "monotonic": 52169.550216041, "phase": "worker_stopped", "firing": [ "MessengerReadyBurnFast", "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 8.0, + "queue_age": 32.172568, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:19:35.732760+00:00", - "monotonic": 49347.604713666, - "phase": "recovery", + "time": "2026-11-06T07:06:42.757662+00:00", + "monotonic": 52174.600673666, + "phase": "worker_stopped", "firing": [ "MessengerReadyBurnFast", "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 8.0, + "queue_age": 37.20728, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:19:40.766904+00:00", - "monotonic": 49352.638829583, - "phase": "recovery", + "time": "2026-11-06T07:06:47.807056+00:00", + "monotonic": 52179.650017583, + "phase": "worker_stopped", "firing": [ "MessengerReadyBurnFast", "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnFast", + "MessengerReadyBurnSlow" + ], + "queue_depth": 8.0, + "queue_age": 42.242111, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:19:45.800827+00:00", - "monotonic": 49357.672704333, + "time": "2026-11-06T07:06:52.863166+00:00", + "monotonic": 52184.706071666, + "phase": "worker_stopped", + "firing": [ + "MessengerReadyBurnSlow" + ], + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 8.0, + "queue_age": 47.286266, + "eligible": 25.0, + "bad": 24.0 + }, + { + "time": "2026-11-06T07:06:57.078846+00:00", + "monotonic": 52188.921692166, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 8.0, + "queue_age": 52.325419, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:19:50.839444+00:00", - "monotonic": 49362.711278166, + "time": "2026-11-06T07:07:02.119205+00:00", + "monotonic": 52193.962019291, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerQueueLag", + "MessengerReadyBurnSlow" + ], + "queue_depth": 6.0, + "queue_age": 57.330259, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:19:55.882108+00:00", - "monotonic": 49367.753864791, + "time": "2026-11-06T07:07:07.168689+00:00", + "monotonic": 52199.011455875, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:20:00.935125+00:00", - "monotonic": 49372.806871, + "time": "2026-11-06T07:07:12.218862+00:00", + "monotonic": 52204.061583875, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:20:05.976123+00:00", - "monotonic": 49377.847798625, + "time": "2026-11-06T07:07:17.266525+00:00", + "monotonic": 52209.109195083, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:20:11.006201+00:00", - "monotonic": 49382.877811166, + "time": "2026-11-06T07:07:22.324719+00:00", + "monotonic": 52214.16734425, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [ + "MessengerReadyBurnSlow" + ], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:20:16.034665+00:00", - "monotonic": 49387.906272875, + "time": "2026-11-06T07:07:27.378443+00:00", + "monotonic": 52219.221010083, "phase": "recovery", "firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 - }, - { - "time": "2026-11-06T06:20:21.073601+00:00", - "monotonic": 49392.945128916, - "phase": "recovery", - "firing": [ + "all_firing": [ "MessengerReadyBurnSlow" ], - "eligible": 17.0, - "bad": 12.0 + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 }, { - "time": "2026-11-06T06:20:26.112334+00:00", - "monotonic": 49397.983810916, + "time": "2026-11-06T07:07:32.422103+00:00", + "monotonic": 52224.264601875, "phase": "recovery", "firing": [], - "eligible": 17.0, - "bad": 12.0 + "all_firing": [], + "queue_depth": 0.0, + "queue_age": 0.0, + "eligible": 25.0, + "bad": 24.0 } ], "jobs": 8, "accepted_at": { - "time": "2026-11-06T06:18:49.190891+00:00", - "monotonic": 49301.063291333 + "time": "2026-11-06T07:05:57.337134+00:00", + "monotonic": 52129.180572833 }, "worker_started": { - "time": "2026-11-06T06:19:31.951591+00:00", - "monotonic": 49343.823643916 + "time": "2026-11-06T07:06:53.291419+00:00", + "monotonic": 52185.134359875 }, "all_ready": { - "time": "2026-11-06T06:19:35.725614+00:00", - "monotonic": 49347.597564333 + "time": "2026-11-06T07:06:57.073151+00:00", + "monotonic": 52188.915998458 }, "result": "PASS" } diff --git a/evidence/slo-track.md b/evidence/slo-track.md index a24838f..6d2c20e 100644 --- a/evidence/slo-track.md +++ b/evidence/slo-track.md @@ -8,27 +8,31 @@ Испытания на существующем локальном стенде Docker Compose. API: три клиентских потока создавали синтетические сообщения, каждый делал паузу 1 секунду между завершившимися запросами; до воздействия — 60 секунд baseline. На время воздействия таблица `users` блокировалась транзакцией на 75 и 180 секунд, между опытами было 100 секунд восстановления. Это воспроизводимая искусственная блокировка, а не реальный пользовательский трафик. Каждые примерно 5 секунд сценарий запрашивал firing alerts через Grafana/Prometheus и сохранял монотонное время. Начало — момент, когда скрипт увидел lock, конец — возврат транзакции. Эти точки и опрос имеют собственную задержку; полученные длительности — наблюдательные оценки, не точные времена переключения Prometheus. -Исходные агрегаты и временные отметки: [slo-incidents.json](slo-incidents.json). Пересчёт: `python3 student-tests/slo_analyze.py` → [slo-analysis.json](slo-analysis.json). Ошибок опроса не зафиксировано. В ходе длительного опыта наблюдался скачок системных часов UTC: разность некоторых соседних UTC-отметок не равна разности монотонных отметок. Длительности ниже рассчитаны по `monotonic`; скачок также мог повлиять на временные окна Prometheus, поэтому эксперимент следует повторить на системе со стабильными часами. +Повторный опыт 2026-11-06: [сырые отсчёты](slo-incidents.json), [расчёт](slo-analysis.json). Анализатор сравнивает UTC и monotonic; максимальное расхождение при округлении до 0,001 с равно 0,000 с, максимальный фактический интервал опроса — 5,037 с. Ошибок опроса — 0. Прежние результаты с расхождением до 83,075 с сохранены в [pre-review](pre-review/slo-analysis.json) и не используются для выводов о длительностях; причина расхождения не установлена. ## Результаты API -| Воздействие | Наблюдаемая длительность блокировки | Fast: первое firing / снятие после конца | Slow: первое firing / снятие после конца | +Все интервалы ниже — разности `monotonic` из JSON; это времена наблюдения, а не точные моменты вычисления Prometheus. Погрешность опроса ограничивается фактическими соседними пробами, до 5,037 с в этом опыте. + +| Воздействие | Наблюдаемая длительность | Fast: обнаружение / снятие после конца | Slow: обнаружение / снятие после конца | | --- | ---: | ---: | ---: | -| Короткое | 75,033 с | 55,088 с / 15,181 с | 80,179 с / 15,181 с | -| Длительное | 180,134 с | 50,278 с / 10,648 с | 90,428 с / 30,724 с | +| Короткое | 75,001 с | 54,676 с / 14,760 с | Не наблюдался | +| Длительное | 180,097 с | 49,860 с / 15,301 с | 90,013 с / 40,394 с | -В коротком сценарии slow впервые появился уже через 5,145 с после окончания воздействия и исчез вместе с fast. По строгой классификации «блокировка уже завершилась — система восстановлена» зафиксирована одна новая тревога вне воздействия; по смыслу окна это запоздалый сигнал о предыдущем расходе бюджета. При длительном сценарии оба сигнала впервые появились во время блокировки. За один baseline получено 13 проб опроса и 0 проб с burn alert: наблюдаемых ложных срабатываний на этом коротком интервале нет, но данных недостаточно для оценки долгосрочной частоты ложных тревог. +В 13 пробах baseline не было burn alerts; новые срабатывания после окончания воздействия в этом повторе не наблюдались. Это ограниченная проверка ложных тревог, не долгосрочная частота. В предыдущем опыте slow впервые появился в восстановлении; повтор не даёт основания обещать отсутствие таких запоздалых сигналов при другом положении окон. -Разность клиентских накопленных счётчиков между первой и последней пробами фазы: baseline 179 успешных / 0 ошибок; короткая блокировка 0 / 33; длительная 0 / 87. Они не включают полные пограничные интервалы и могут включать timeout; это не числитель HTTP 5xx SLI. Срабатывания рассчитаны Prometheus по серверным метрикам. В Loki найден синтетический HTTP 503 с trace `47caf6293f35aabf09f71f307ed860a3`, где span `postgres.user_for_token` занял 5,004 с: это проверка пути диагностики в runbook. Проверка выполнена автором сценария; прохождение другим студентом пока не зафиксировано. +Для диагностики сохранён [лог 503 и связанный trace](review-db-trace.json): `postgres.user_for_token` занимает 5,006 с, HTTP span — 5,009 с (разности `endTimeUnixNano-startTimeUnixNano`, делённые на 10⁹, округлены до 0,001 с). Это подтверждает задержку БД, а не только медленный HTTP handler. Проверку выполнял автор сценария; независимое прохождение runbook другим студентом не подтверждено. ## Проверка SLO вложений `python3 student-tests/slo_ready_incident.py` убедился, что незавершённых задач нет, остановил worker, принял восемь синтетических вложений и дождался alert. Затем worker запущен, все вложения достигли ready. Исходные измерения: [slo-ready-incident.json](slo-ready-incident.json). -До воздействия счётчики matured/плохих заданий были 9/4. В первой пробе firing — 17/12: прирост обоих счётчиков равен 8, то есть все восемь заданий попали в знаменатель и нарушили 30-секундный срок. Fast и slow стали firing через 40,279 с после последнего принятия. Все задания были ready через 3,774 с после команды восстановления worker. Fast исчез через 13,849 с, slow — через 54,160 с после команды запуска. Значения опросные, включая задержку запуска Docker и опроса. При снятии slow новых заданий не поступало: условие минимального числа событий перестало выполняться. Это не доказывает, что 24-часовой SLO снова выполнен; факт просрочки остаётся в счётчиках. +Повтор после исправления методики: baseline не имел firing alerts, queue depth/age были 0. Счётчики matured/bad — 17/16 до опыта и 25/24 после: прирост каждого `25−17 = 24−16 = 8`, все восемь новых заданий нарушили срок. Fast и slow впервые наблюдались через 40,370 с после последнего принятия; QueueLag — через 55,525 с. Все файлы стали ready через 3,782 с после команды запуска worker. QueueLag снялся через 13,877 с, slow — через 39,130 с после запуска. Fast уже отсутствовал в последней пробе до запуска worker (за 0,428 с до отметки запуска): прекращение firing не означало устранения причины. Финальные depth/age = 0, bad остался 24. Все интервалы получены вычитанием сохранённых `monotonic`, округлением до 0,001 с. Максимальный фактический шаг опроса в ready-опыте — 5,299 с; максимальное расхождение приращений UTC/monotonic меньше 0,001 с. + +[Промежуточный неудачный прогон](pre-review/ready-scenario-failed.json) выявил ошибки теста: он требовал одновременного firing Fast и QueueLag и выбирал первый series старого outbox. Исправленный сценарий фиксирует каждый alert отдельно, проверяет восстановление обоих и использует max абсолютного счётчика общей DB-когорты по экземплярам outbox. Неудачный прогон не скрыт и не считается PASS. ## Дополнительная проверка правил и ограничения `promtool test rules` на [burn_rules.test.yaml](../student-tests/burn_rules.test.yaml) проверил нормальную нагрузку без burn alerts, тяжёлый краткий всплеск с последующим снятием fast, устойчивую 20%-ную долю ошибок с firing slow без fast, а также поведение при малом числе событий и fast alert просроченных вложений. Это синтетические ряды, не замена эксперименту с реальными данными. -Границы: API SLI не включает ошибки LB до приложения. Счётчики готовности вычисляются из состояний PostgreSQL раз в цикл outbox; при недоступном outbox данные устаревают, а события во время разрыва наблюдения могут не войти в `increase` после перезапуска процесса. Начало и конец срока фиксируются внутри DB-транзакций, поэтому приближённо описывают пользовательские границы durable acceptance и ready. Возможные manual replay/редактирование БД, смена или удаление заданий могут нарушить монотонность DB-счётчиков; учебный API не удаляет задания. Нет оценки ложных срабатываний за длительный период или проверки runbook другим студентом. Ошибки, произошедшие до восстановления, учитываются в бюджетном окне до истечения окна и могут вызвать задержанный alert. +Границы: API SLI не включает ошибки LB до приложения. Ready-когорта после миграции 009 использует сохранённые commit timestamps и неизменный результат; старые задания исключены. При недоступном outbox данные устаревают; stale/unknown alerts показывают неполноту измерения. События во время разрыва наблюдения могут не войти в `increase` после смены процесса. Короткий опыт не доказывает выполнение 24-часового SLO. Проверка другим студентом отсутствует; протокол описан в runbook. diff --git a/infra/prometheus/alerts.yaml b/infra/prometheus/alerts.yaml index d7a700b..af62567 100644 --- a/infra/prometheus/alerts.yaml +++ b/infra/prometheus/alerts.yaml @@ -1,6 +1,16 @@ groups: - name: messenger rules: + - alert: MessengerQueueTelemetryStale + expr: time() - max(messenger_queue_sample_time_seconds{service_name="messenger-outbox"}) > 20 or sum(absent(messenger_queue_sample_time_seconds{service_name="messenger-outbox"})) + for: 15s + labels: {severity: warning} + annotations: {summary: 'Queue and ready SLO telemetry is stale', runbook: 'See docs/slo-runbook.md; check outbox and DB before interpreting ready SLO.'} + - alert: MessengerReadyTimestampUnknown + expr: max(messenger_slo_unknown{service_name="messenger-outbox"}) > 0 + for: 15s + labels: {severity: warning} + annotations: {summary: 'Ready SLO commit timestamps unavailable', runbook: 'Check track_commit_timestamp and outbox; unknown events are not proof of SLO compliance.'} - alert: MessengerHighErrorRate expr: (sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m])), 0.001) > 0.05 for: 10s diff --git a/migrations/009_slo_commits.sql b/migrations/009_slo_commits.sql new file mode 100644 index 0000000..dd5f37c --- /dev/null +++ b/migrations/009_slo_commits.sql @@ -0,0 +1,11 @@ +-- A separate immutable cohort: replay and job cleanup must not rewrite SLO history. +-- Existing jobs have no recoverable acceptance commit and are deliberately not backfilled. +CREATE TABLE attachment_slo ( + attachment_id uuid PRIMARY KEY, + accepted_xid xid NOT NULL DEFAULT pg_current_xact_id()::xid, + ready_xid xid, + accepted_at timestamptz, + ready_at timestamptz, + bad boolean +); +CREATE INDEX attachment_slo_unresolved_idx ON attachment_slo(accepted_at) WHERE bad IS NULL; diff --git a/pyproject.toml b/pyproject.toml index 6d3ce68..df9aa92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] -name = "mtusi2026lab4" -version = "0.1.0" -description = "MTUSI high-load messenger, laboratory 4" +name = "mtusi2026lab6" +version = "1.6.0" +description = "MTUSI high-load messenger, laboratory 6" readme = "README.md" requires-python = ">=3.12" dependencies = [ diff --git a/src/broker.py b/src/broker.py index fadd839..ed306b5 100644 --- a/src/broker.py +++ b/src/broker.py @@ -48,7 +48,11 @@ async def publish(payload: dict, context=None) -> None: try: with tel.tracer.start_as_current_span("attachment.publish", context=tel.extract(context), kind=tel.SpanKind.PRODUCER, record_exception=False, set_status_on_exception=False) as span: span.set_attribute("messaging.retry.attempt", int(payload.get("attempt", 1))) - await run_in_threadpool(_publish, payload, tel.carrier()) + try: + await run_in_threadpool(_publish, payload, tel.carrier()) + except Exception: + span.set_status(tel.Status(tel.StatusCode.ERROR)) + raise except (pika.exceptions.AMQPError, OSError, ValueError) as error: raise ApiError(503, "unavailable", "Брокер недоступен") from error diff --git a/src/main.py b/src/main.py index a8cb62d..004b3e6 100644 --- a/src/main.py +++ b/src/main.py @@ -75,7 +75,7 @@ async def lifespan(application: FastAPI): app = FastAPI( title="MTUSI Messenger", - version="1.4.0", + version="1.6.0", redoc_url=None, lifespan=lifespan, ) @@ -97,14 +97,15 @@ async def add_request_id(request: Request, call_next) -> Response: return response finally: route = getattr(request.scope.get("route"), "path", "unmatched") - attrs = {"method": request.method, "route": route, "status_class": f"{status_code // 100}xx"} - span.update_name(f"{request.method} {route}") - span.set_attributes({"http.request.method": request.method, "http.route": route, "http.response.status_code": status_code}) + method = tel.http_method(request.method) + attrs = {"method": method, "route": route, "status_class": f"{status_code // 100}xx"} + span.update_name(f"{method} {route}") + span.set_attributes({"http.request.method": method, "http.route": route, "http.response.status_code": status_code}) if status_code >= 500: span.set_status(tel.Status(tel.StatusCode.ERROR)) tel.http_requests.add(1, attrs) tel.http_duration.record(time.monotonic() - started_at, attrs) - tel.log.info("http.request", extra={"request_id": request.state.request_id, "method": request.method, "route": route, "status": status_code}) + tel.log.info("http.request", extra={"request_id": request.state.request_id, "method": method, "route": route, "status": status_code}) def error_response( diff --git a/src/outbox.py b/src/outbox.py index 79522aa..a8ac14d 100644 --- a/src/outbox.py +++ b/src/outbox.py @@ -3,6 +3,7 @@ import asyncio import logging import os import json +import time from .broker import publish from .config import load_settings @@ -26,20 +27,16 @@ async def run() -> None: context = event['trace_context'] await publish(event["payload"], json.loads(context) if isinstance(context, str) else context) await store.mark_outbox_published(event["id"]) - try: - queue = await store.pool.fetchrow("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)), 0)::float8 AS age FROM attachment_jobs WHERE status='queued'") - slo = await store.pool.fetchrow(""" - SELECT count(*) FILTER (WHERE created_at <= now() - interval '30 seconds') AS eligible, - count(*) FILTER (WHERE created_at <= now() - interval '30 seconds' - AND (status <> 'ready' OR updated_at > created_at + interval '30 seconds')) AS bad - FROM attachment_jobs - """) - tel.state.update(queue_depth=queue['depth'], queue_age=queue['age'], - slo_ready_eligible=slo['eligible'], slo_ready_bad=slo['bad']) - except Exception: - logging.warning("queue telemetry unavailable") except Exception: logging.exception("outbox dispatch failed; durable event remains pending") + try: + queue = await store.pool.fetchrow("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)), 0)::float8 AS age FROM attachment_jobs WHERE status='queued'") + slo = await store.sample_ready_slo() + tel.state.update(queue_depth=queue['depth'], queue_age=queue['age'], + slo_ready_eligible=slo['eligible'], slo_ready_bad=slo['bad'], + slo_unknown=slo['unknown'], queue_sample_time=time.time()) + except Exception: + logging.warning("queue telemetry unavailable") await asyncio.sleep(interval) finally: await store.close() diff --git a/src/storage/postgres.py b/src/storage/postgres.py index b4cedd9..9e6edfe 100644 --- a/src/storage/postgres.py +++ b/src/storage/postgres.py @@ -341,6 +341,9 @@ class PostgresStore: await (connection or self.pool).execute( "INSERT INTO attachment_jobs (job_id, attachment_id, trace_context) VALUES ($1, $2, $3::jsonb)", job_id, attachment_id, json.dumps(tel.carrier()) ) + await (connection or self.pool).execute( + "INSERT INTO attachment_slo (attachment_id) VALUES ($1)", attachment_id + ) payload = { "schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1", "attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1, @@ -438,10 +441,44 @@ class PostgresStore: attachment_id, original_key, ) await connection.execute("UPDATE attachment_jobs SET status = 'ready', lease_until = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id) + await connection.execute( + "UPDATE attachment_slo SET ready_xid=pg_current_xact_id()::xid " + "WHERE attachment_id=$1 AND ready_xid IS NULL", attachment_id + ) return True except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error + async def ready_seconds(self, attachment_id: UUID): + # Both ends are DB commit timestamps, not transaction start or a worker's clock. + return await self.pool.fetchval(""" + SELECT EXTRACT(EPOCH FROM ( + COALESCE(ready_at, pg_xact_commit_timestamp(ready_xid)) - + COALESCE(accepted_at, pg_xact_commit_timestamp(accepted_xid))))::float8 + FROM attachment_slo WHERE attachment_id=$1 + """, attachment_id) + + async def sample_ready_slo(self): + # Materialize promptly: PostgreSQL does not retain commit timestamps forever. + async with self.pool.acquire() as connection, connection.transaction(): + await connection.execute(""" + UPDATE attachment_slo SET + accepted_at=COALESCE(accepted_at, pg_xact_commit_timestamp(accepted_xid)), + ready_at=COALESCE(ready_at, pg_xact_commit_timestamp(ready_xid)) + WHERE accepted_at IS NULL OR (ready_xid IS NOT NULL AND ready_at IS NULL) + """) + await connection.execute(""" + UPDATE attachment_slo SET bad=(ready_at IS NULL OR ready_at>accepted_at+interval '30 seconds') + WHERE bad IS NULL AND accepted_at<=now()-interval '30 seconds' + AND (ready_xid IS NULL OR ready_at IS NOT NULL) + """) + return await connection.fetchrow(""" + SELECT count(bad) AS eligible, count(*) FILTER (WHERE bad) AS bad, + count(*) FILTER (WHERE accepted_at IS NULL OR + (ready_xid IS NOT NULL AND ready_at IS NULL)) AS unknown + FROM attachment_slo + """) + async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int, delay_seconds: float, claim_time, permanent_code=None) -> bool: """Return true only when a bounded at-least-once retry was recorded.""" try: diff --git a/src/telemetry.py b/src/telemetry.py index c8c2969..082661e 100644 --- a/src/telemetry.py +++ b/src/telemetry.py @@ -22,9 +22,9 @@ ready_duration = meter.create_histogram("messenger.job.ready.duration", unit="s" busy = meter.create_up_down_counter("messenger.worker.busy", unit="{worker}") capacity = meter.create_up_down_counter("messenger.worker.capacity", unit="{worker}") state = {"queue_depth": 0, "queue_age": 0, "pool_used": 0, "pool_capacity": 0, "pool_waiting": 0, - "slo_ready_eligible": 0, "slo_ready_bad": 0} -for name, unit in (("queue_depth", "{job}"), ("queue_age", "s"), ("pool_used", "{connection}"), ("pool_capacity", "{connection}"), ("pool_waiting", "{request}")): - meter.create_observable_gauge("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key])]], unit=unit) + "slo_ready_eligible": 0, "slo_ready_bad": 0, "slo_unknown": 0, "queue_sample_time": 0} +for name, unit in (("queue_sample_time", "s"), ("slo_unknown", "{job}"), ("queue_depth", "{job}"), ("queue_age", "s"), ("pool_used", "{connection}"), ("pool_capacity", "{connection}"), ("pool_waiting", "{request}")): + meter.create_observable_gauge("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key], {"queue": "attachment.process.v1"} if key.startswith("queue_") else {})]], unit=unit) for name in ("slo_ready_eligible", "slo_ready_bad"): meter.create_observable_counter("messenger." + name, callbacks=[lambda options, key=name: [metrics.Observation(state[key])]], unit="{job}") @@ -46,6 +46,29 @@ log = logging.getLogger("messenger.telemetry") log.setLevel(logging.INFO) log.propagate = False +LOG_FIELDS = ("request_id", "method", "route", "status", "outcome") +LOG_EVENTS = {"http.request", "job.processed", "telemetry.unavailable"} +STANDARD_LOG_FIELDS = frozenset(logging.makeLogRecord({}).__dict__) + + +def http_method(value): + return value if value in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "CONNECT", "TRACE"} else "_OTHER" + + +class SafeLogFilter(logging.Filter): + """Sanitize the shared record BEFORE both stdout and OTLP handlers.""" + def filter(self, record): + for key in list(record.__dict__): + if key not in STANDARD_LOG_FIELDS and key not in LOG_FIELDS: + del record.__dict__[key] + record.msg = record.msg if isinstance(record.msg, str) and record.msg in LOG_EVENTS else "telemetry.unavailable" + record.args = () + record.exc_info = record.exc_text = record.stack_info = None + return True + + +log.addFilter(SafeLogFilter()) + class JsonFormatter(logging.Formatter): def format(self, record): ctx = trace.get_current_span().get_span_context() @@ -53,7 +76,7 @@ class JsonFormatter(logging.Formatter): "service.name": os.environ.get("OTEL_SERVICE_NAME", "messenger"), "event": record.getMessage()} if ctx.is_valid: obj.update(trace_id=f"{ctx.trace_id:032x}", span_id=f"{ctx.span_id:016x}") - for field in ("request_id", "method", "route", "status", "outcome"): + for field in LOG_FIELDS: if hasattr(record, field): obj[field] = getattr(record, field) return json.dumps(obj, ensure_ascii=False) diff --git a/src/worker.py b/src/worker.py index ef82c21..6aa4f8b 100644 --- a/src/worker.py +++ b/src/worker.py @@ -94,17 +94,17 @@ def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes ] -async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None: +async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str: # Excludes cleanup for the whole lifetime of S3 effects, including stale attempts. async with store.attachment_transaction(): - await process_locked(store, s3, payload, lease_seconds) + return await process_locked(store, s3, payload, lease_seconds) -async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None: +async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str: attachment_id = UUID(payload["attachment_id"]) attachment = await store.claim_attachment_job(attachment_id, lease_seconds) if attachment is None: - return + return "skipped" payload['_claim_time'] = attachment['claim_time'] payload['attempt'] = attachment.get('attempt', payload.get('attempt', 1)) attempt_key = uuid4().hex @@ -133,13 +133,18 @@ async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease os._exit(70) committed = await store.complete_attachment_job(attachment_id, original_key, metadata, attachment['claim_time']) if committed: - if attachment.get('created_at') is not None: - tel.ready_duration.record(max(0, (datetime.now(timezone.utc) - attachment['created_at']).total_seconds()), {"job_type": "attachment"}) + try: + duration = await store.ready_seconds(attachment_id) + if duration is not None: + tel.ready_duration.record(duration, {"job_type": "attachment"}) + except Exception: + tel.log.warning("telemetry.unavailable") try: await s3.delete(attachment["staging_key"]) except Exception: logging.warning('staging cleanup deferred attachment_id=%s', attachment_id) + return "success" if committed else "skipped" def settle_delivery(channel, delivery_tag, future): """Only a successful durable disposition permits acknowledgement.""" @@ -169,7 +174,7 @@ def main() -> None: s3 = S3Store(S3Settings.load()) await store.connect() try: - await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))) + return await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))) finally: s3.client.close() await store.close() @@ -198,15 +203,15 @@ def main() -> None: except (ValueError, KeyError, TypeError): try: asyncio.run(publish_dlq({'failure': 'invalid_job'})) - return True, False + return True, "failed" except Exception: - return False, False + return False, "failed" try: - asyncio.run(handle_async(payload)) + outcome = asyncio.run(handle_async(payload)) except Exception as error: logging.warning("job failed attachment_id=%s", payload['attachment_id']) if '_claim_time' not in payload: - return False, False + return False, "failed" try: attachment_id = UUID(payload["attachment_id"]) attempt = int(payload.get("attempt", 1)) @@ -214,17 +219,16 @@ def main() -> None: asyncio.run(schedule_retry(attachment_id, delay, payload['_claim_time'], str(error) if isinstance(error, PermanentFailure) else None)) except Exception: - return False, False - return True, False - return True, True + return False, "failed" + return True, "failed" + return True, outcome def handle(body, headers): started = time.monotonic() tel.busy.add(1) with tel.tracer.start_as_current_span("attachment.consume", context=tel.extract(headers), kind=tel.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span: try: - acknowledged, succeeded = handle_job(body) - outcome = "success" if succeeded else "failed" + acknowledged, outcome = handle_job(body) return acknowledged except Exception: outcome = "failed" diff --git a/student-tests/capture_observability.py b/student-tests/capture_observability.py new file mode 100644 index 0000000..f0d066c --- /dev/null +++ b/student-tests/capture_observability.py @@ -0,0 +1,41 @@ +"""Export synthetic trace/log evidence through Grafana; credentials never enter artifacts.""" +import argparse +import base64 +from datetime import datetime, timezone +import json +from pathlib import Path +import urllib.parse +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +auth = 'Basic ' + base64.b64encode(('admin:' + (ROOT/'secrets/grafana_password').read_text().strip()).encode()).decode() +def get(source, path, params=None): + url = f'http://127.0.0.1:3000/api/datasources/proxy/uid/{source}/' + path + if params: url += '?' + urllib.parse.urlencode(params) + with urllib.request.urlopen(urllib.request.Request(url,headers={'Authorization':auth}),timeout=15) as r: + return json.load(r) + +def capture(trace_id=None, query=None): + query = query or '{service_name="messenger-api"} | status="503"' + if trace_id: + query = '{service_name=~"messenger-api|messenger-worker"} | trace_id="' + trace_id + '"' + logs = get('loki','loki/api/v1/query_range',{'query':query,'since':'30m','limit':20,'direction':'backward'}) + if not trace_id: + for stream in logs['data']['result']: + for value in stream['values']: + metadata = dict(stream['stream']) + if len(value)>2: metadata.update(value[2]) + trace_id = metadata.get('trace_id') + if trace_id: break + if trace_id: break + if not trace_id: + raise RuntimeError('No correlated log/trace found') + trace = get('tempo','api/traces/'+trace_id) + return {'captured_at':datetime.now(timezone.utc).isoformat(),'query':query,'trace_id':trace_id,'logs':logs,'trace':trace} + +if __name__=='__main__': + parser=argparse.ArgumentParser();parser.add_argument('--trace-id');parser.add_argument('--output',required=True) + args=parser.parse_args() + result=capture(args.trace_id) + (ROOT/args.output).write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n') + print('Saved correlated evidence:',args.output,'trace_id:',result['trace_id']) diff --git a/student-tests/check_dashboard.py b/student-tests/check_dashboard.py new file mode 100644 index 0000000..cef28a8 --- /dev/null +++ b/student-tests/check_dashboard.py @@ -0,0 +1,18 @@ +"""Execute every provisioned PromQL target against the running Grafana datasource.""" +import importlib.util +import json +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('capture',ROOT/'student-tests/capture_observability.py') +capture=importlib.util.module_from_spec(spec);spec.loader.exec_module(capture) +dashboard=json.loads((ROOT/'infra/grafana/dashboards/messenger.json').read_text()) +results=[] +for panel in dashboard['panels']: + for target in panel.get('targets',[]): + if 'expr' not in target:continue + query=target['expr'].replace('$__rate_interval','1m').replace('$__interval','5s') + result=capture.get('prometheus','api/v1/query',{'query':query}) + assert result['status']=='success',result + results.append({'panel':panel['title'],'query':query,'series':len(result['data']['result'])}) +(ROOT/'evidence/review-dashboard.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n') +print('PASS:',len(results),'dashboard queries; empty targets:',sum(r['series']==0 for r in results)) diff --git a/student-tests/check_retry_trace.py b/student-tests/check_retry_trace.py new file mode 100644 index 0000000..1f75474 --- /dev/null +++ b/student-tests/check_retry_trace.py @@ -0,0 +1,31 @@ +"""Verify exported parent relationships and the completed retry, not just trace existence.""" +import base64 +import json +from collections import Counter +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +data=json.loads((ROOT/'evidence/review-retry-trace.json').read_text()) +spans=[s for batch in data['trace']['batches'] for scope in batch.get('scopeSpans',[]) for s in scope.get('spans',[])] +by_id={s['spanId']:s for s in spans} +counts=Counter(s['name'] for s in spans) +producers=[s for s in spans if s['kind']=='SPAN_KIND_PRODUCER'] +consumers=[s for s in spans if s['kind']=='SPAN_KIND_CONSUMER'] +assert len(producers)>=2 and consumers,counts +for producer in producers: + ancestor = producer + for _ in range(len(spans)): + ancestor = by_id[ancestor['parentSpanId']] + if ancestor['kind']=='SPAN_KIND_SERVER': break + else: raise AssertionError('producer has no API ancestor') +for consumer in consumers: + assert by_id[consumer['parentSpanId']]['kind']=='SPAN_KIND_PRODUCER' +assert counts['postgres.complete_attachment_job']>=1,counts +assert any(s['name'].startswith('s3.') for s in spans),counts +assert all(base64.b64decode(s['traceId']).hex()==data['trace_id'] for s in spans) +services={stream['stream']['service_name'] for stream in data['logs']['data']['result']} +assert {'messenger-api','messenger-worker'}<=services,services +result={'trace_id':data['trace_id'],'span_count':len(spans),'span_names':dict(counts), + 'producer_count':len(producers),'consumer_count':len(consumers),'log_services':sorted(services), + 'parent_relationships':'PASS','note':'Abrupt os._exit may lose unfinished spans of the crashed attempt; redelivery and committed retry remain visible.'} +(ROOT/'evidence/review-retry-summary.json').write_text(json.dumps(result,indent=2)+'\n') +print('PASS:',json.dumps(result)) diff --git a/student-tests/pipeline_critical.py b/student-tests/pipeline_critical.py index dfb716f..3750ba9 100644 --- a/student-tests/pipeline_critical.py +++ b/student-tests/pipeline_critical.py @@ -53,12 +53,14 @@ class CriticalTests(unittest.IsolatedAsyncioTestCase): 'content_type': 'image/png', 'size_bytes': 1, 'sha256': 'unused', } store.complete_attachment_job.return_value = committed + store.ready_seconds.return_value = 0.25 s3.settings = SimpleNamespace(max_pixels=40000000) s3.download_key.return_value = b'fixture' s3.delete.side_effect = RuntimeError('storage temporarily unavailable') payload = {'attachment_id': str(uuid4())} with patch('src.worker.render_isolated', return_value=[('thumbnail', b'jpeg', 'image/jpeg', 1, 1)]): - await process(store, s3, payload, 60) + outcome = await process(store, s3, payload, 60) + self.assertEqual(outcome, "success" if committed else "skipped") self.assertEqual(store.complete_attachment_job.call_args.args[-1], stamp) return store, s3 @@ -71,6 +73,14 @@ class CriticalTests(unittest.IsolatedAsyncioTestCase): _, s3 = await self.run_processing(False) s3.delete.assert_not_awaited() + async def test_duplicate_is_skipped_not_successful(self): + store, s3 = AsyncMock(), AsyncMock() + store.claim_attachment_job.return_value = None + result = await process(store, s3, {'attachment_id': str(uuid4())}, 60) + self.assertEqual(result, 'skipped') + store.complete_attachment_job.assert_not_awaited() + s3.copy.assert_not_awaited() + async def test_stale_claim_cannot_publish_metadata(self): store = PostgresStore('unused') connection = AsyncMock() diff --git a/student-tests/retry_evidence.py b/student-tests/retry_evidence.py new file mode 100644 index 0000000..f3b5b01 --- /dev/null +++ b/student-tests/retry_evidence.py @@ -0,0 +1,30 @@ +"""Crash one idle lab worker after S3 writes; restore normal configuration in finally.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +def compose(*args, env=None): + return subprocess.run(['docker','compose',*args],cwd=ROOT,env=env,check=True,capture_output=True,text=True).stdout +count=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',"SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')") +if int(count.strip()):raise RuntimeError('Pending work; refusing crash injection') +try: + compose('stop','-t','5','worker') + compose('up','-d','--no-deps','worker',env={**os.environ,'WORKER_FAULT_AFTER_S3_PUT':'1'}) + subprocess.run([sys.executable,'student-tests/trace_retry.py'],cwd=ROOT,check=True,timeout=180) + out=json.loads((ROOT/'evidence/review-retry-job.json').read_text()) + from uuid import UUID + identity=str(UUID(out['attachment_id'])) + out['db_attempt']=int(compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',f"SELECT attempt FROM attachment_jobs WHERE attachment_id='{identity}'").strip()) + cid=compose('ps','-q','worker').strip() + out['restart_count']=int(subprocess.run(['docker','inspect','--format','{{.RestartCount}}',cid],check=True,capture_output=True,text=True).stdout.strip()) + assert out['db_attempt']==2 and out['restart_count']==1,out + (ROOT/'evidence/review-retry-job.json').write_text(json.dumps(out,indent=2)+'\n') + time.sleep(10) + subprocess.run([sys.executable,'student-tests/capture_observability.py','--trace-id',out['trace_id'],'--output','evidence/review-retry-trace.json'],cwd=ROOT,check=True) +finally: + compose('stop','-t','5','worker') + compose('up','-d','--no-deps','worker',env={**os.environ,'WORKER_FAULT_AFTER_S3_PUT':'0'}) +print('PASS: attempt 2, one restart, normal worker restored') diff --git a/student-tests/review_checks.py b/student-tests/review_checks.py new file mode 100644 index 0000000..a22b16c --- /dev/null +++ b/student-tests/review_checks.py @@ -0,0 +1,22 @@ +"""Repeat short review checks; experiments are separate and intentionally disruptive.""" +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +ROOT=Path(__file__).resolve().parents[1] +checks=[['make','check'],['make','validate','PYTHON=.venv/bin/python'], + ['docker','compose','config','--quiet'], + ['docker','run','--rm','-v',f'{ROOT}:/workspace:ro','--entrypoint','promtool','prom/prometheus:v3.2.1','test','rules', + '/workspace/student-tests/alerts.test.yaml','/workspace/student-tests/burn_rules.test.yaml','/workspace/student-tests/telemetry_alerts.test.yaml'], + ['make','test'],['python3','student-tests/check_dashboard.py']] +results=[] +for command in checks: + proc=subprocess.run(command,cwd=ROOT,capture_output=True,text=True,timeout=180) + results.append({'command':command,'exit_code':proc.returncode,'output':proc.stdout+proc.stderr}) +for filename in ('telemetry_unit.py','pipeline_critical.py','slo_commit_integration.py'): + command=['docker','compose','exec','-T','api-a','/app/.venv/bin/python','-'] + proc=subprocess.run(command,input=(ROOT/'student-tests'/filename).read_text(),cwd=ROOT,capture_output=True,text=True,timeout=60) + results.append({'command':command,'stdin':f'student-tests/{filename}','exit_code':proc.returncode,'output':proc.stdout+proc.stderr}) +(ROOT/'evidence/review-checks.json').write_text(json.dumps({'utc':datetime.now(timezone.utc).isoformat(),'checks':results},ensure_ascii=False,indent=2)+'\n') +for r in results:print('PASS' if r['exit_code']==0 else 'FAIL',r.get('stdin',r['command'])) +raise SystemExit(any(r['exit_code'] for r in results)) diff --git a/student-tests/review_queries.py b/student-tests/review_queries.py new file mode 100644 index 0000000..1c4d9ae --- /dev/null +++ b/student-tests/review_queries.py @@ -0,0 +1,13 @@ +"""Read current health of telemetry, without printing access credentials.""" +import importlib.util +import json +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('capture',ROOT/'student-tests/capture_observability.py') +c=importlib.util.module_from_spec(spec);spec.loader.exec_module(c) +queries=['messenger_slo_unknown{service_name="messenger-outbox"}', + 'time()-max(messenger_queue_sample_time_seconds{service_name="messenger-outbox"})', + 'messenger_queue_depth{service_name="messenger-outbox"}', + 'messenger_job_ready_duration_seconds_count{service_name="messenger-worker"}'] +for q in queries: + print(q,json.dumps(c.get('prometheus','api/v1/query',{'query':q})['data']['result'])) diff --git a/student-tests/review_resilience.py b/student-tests/review_resilience.py new file mode 100644 index 0000000..cfc49e1 --- /dev/null +++ b/student-tests/review_resilience.py @@ -0,0 +1,20 @@ +"""Run inherited public durability/failover checks and restore both API replicas.""" +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +ROOT=Path(__file__).resolve().parents[1] +results=[] +try: + for command in [ + ['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'], + ['make','test-failover','ACTION_SCRIPT=scripts/stop-one.sh']]: + result=subprocess.run(command,cwd=ROOT,capture_output=True,text=True,timeout=240) + results.append({'command':command,'exit_code':result.returncode,'output':result.stdout+result.stderr}) + print('PASS' if result.returncode==0 else 'FAIL',command,flush=True) + if result.returncode:break +finally: + restore=subprocess.run(['docker','compose','up','-d','--no-deps','--wait','api-a','api-b','worker','outbox'],cwd=ROOT,capture_output=True,text=True,timeout=90) + results.append({'restore_exit_code':restore.returncode,'output':restore.stdout+restore.stderr}) + (ROOT/'evidence/review-resilience.json').write_text(json.dumps({'utc':datetime.now(timezone.utc).isoformat(),'checks':results},ensure_ascii=False,indent=2)+'\n') +raise SystemExit(any(r.get('exit_code',r.get('restore_exit_code')) for r in results)) diff --git a/student-tests/slo_analyze.py b/student-tests/slo_analyze.py index e4dfb87..d01f1d4 100644 --- a/student-tests/slo_analyze.py +++ b/student-tests/slo_analyze.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Summarize observed alert transitions; sampling bounds detection precision.""" import json +from datetime import datetime from pathlib import Path root = Path(__file__).resolve().parents[1] @@ -8,6 +9,13 @@ data = json.loads((root / 'evidence/slo-incidents.json').read_text()) samples = sorted((s for s in data['samples'] if 'firing' in s), key=lambda s: s['monotonic']) alerts = ['MessengerApiBurnFast', 'MessengerApiBurnSlow'] output = {'sampling_interval_seconds': 5, 'monitor_errors': sum('monitor_error' in s for s in data['samples']), 'phases': {}} +drifts = [abs((datetime.fromisoformat(b['time']) - datetime.fromisoformat(a['time'])).total_seconds() + - (b['monotonic'] - a['monotonic'])) for a, b in zip(samples, samples[1:])] +gaps = [b['monotonic'] - a['monotonic'] for a, b in zip(samples, samples[1:])] +output['timing_quality'] = {'max_wall_monotonic_disagreement_seconds': round(max(drifts, default=0), 3), + 'max_observed_sampling_gap_seconds': round(max(gaps, default=0), 3), + 'valid_for_elapsed_time_claims': max(drifts, default=0) <= 1, + 'note': 'Disagreement does not identify its cause; rerun if invalid. Precision is bounded by actual gaps, not the configured sleep.'} base = [s for s in samples if s['phase'] == 'baseline'] output['baseline'] = {'samples': len(base), 'samples_with_burn_alert': sum(bool(s['firing']) for s in base), 'scope': 'one observed healthy baseline, not a long-term false-positive rate'} for name, period in data['periods'].items(): diff --git a/student-tests/slo_commit_integration.py b/student-tests/slo_commit_integration.py new file mode 100644 index 0000000..acc1dd0 --- /dev/null +++ b/student-tests/slo_commit_integration.py @@ -0,0 +1,45 @@ +"""Run inside api-a; session-local table shadows real SLO data (no business mutations).""" +import asyncio +from datetime import datetime, timezone +from uuid import uuid4 +import asyncpg +from src.config import load_settings +from src.storage.postgres import PostgresStore + +async def main(): + store = PostgresStore(load_settings().database_url) + store._pool = await asyncpg.create_pool(load_settings().database_url, min_size=1, max_size=1) + try: + async with store.pool.acquire() as conn: + await conn.execute('CREATE TEMP TABLE attachment_slo (LIKE public.attachment_slo INCLUDING DEFAULTS)') + identity = uuid4() + async with conn.transaction(): + await conn.execute('INSERT INTO attachment_slo (attachment_id) VALUES ($1)', identity) + await asyncio.sleep(.2) + before_accept_commit = await conn.fetchval('SELECT clock_timestamp()') + async with conn.transaction(): + await conn.execute('UPDATE attachment_slo SET ready_xid=pg_current_xact_id()::xid') + await asyncio.sleep(.2) + before_ready_commit = await conn.fetchval('SELECT clock_timestamp()') + await store.sample_ready_slo() + row = await store.pool.fetchrow('SELECT * FROM attachment_slo') + assert row['accepted_at'] >= before_accept_commit + assert row['ready_at'] >= before_ready_commit + elapsed = await store.ready_seconds(identity) + assert elapsed >= .2, elapsed + # Mature a successful synthetic cohort, then emulate replay metadata changes. + await store.pool.execute("UPDATE attachment_slo SET accepted_at=now()-interval '32 seconds',ready_at=now()-interval '31 seconds'") + first = dict(await store.sample_ready_slo()) + assert first == {'eligible': 1, 'bad': 0, 'unknown': 0}, first + await store.pool.execute("UPDATE attachment_slo SET ready_at=now()+interval '1 minute'") + assert dict(await store.sample_ready_slo()) == first + await store.pool.execute("INSERT INTO attachment_slo (attachment_id,accepted_at) VALUES ($1,now()-interval '31 seconds')", uuid4()) + late = dict(await store.sample_ready_slo()) + assert late == {'eligible': 2, 'bad': 1, 'unknown': 0}, late + await store.pool.execute('UPDATE attachment_slo SET ready_at=now(),ready_xid=pg_current_xact_id()::xid WHERE ready_at IS NULL') + assert dict(await store.sample_ready_slo()) == late + print('PASS: commit boundaries; immutable good/bad cohorts; late completion remains bad') + finally: + await store.close() + +asyncio.run(main()) diff --git a/student-tests/slo_incidents.py b/student-tests/slo_incidents.py index 241c504..09c2bcc 100644 --- a/student-tests/slo_incidents.py +++ b/student-tests/slo_incidents.py @@ -2,7 +2,7 @@ """Run two bounded DB-lock incidents against the local Compose project. Creates synthetic data, records only aggregate counts/alert states, and releases -all locks on exit. Run on the isolated lab6 stack, never a shared database. +bounded locks; interrupted server-side sleeps may persist until their deadline. Run on the isolated lab6 stack, never a shared database. """ import base64 from concurrent.futures import ThreadPoolExecutor @@ -43,7 +43,7 @@ def monitor(): try: alerts = get('alerts')['alerts'] firing = sorted(a['labels']['alertname'] for a in alerts if a['state'] == 'firing' and 'Burn' in a['labels']['alertname']) - with LOCK: sample = {'time': utc(), 'monotonic': time.monotonic(), 'phase': SCENARIO['phase'], 'counts': dict(COUNTS), 'firing': firing} + with LOCK: sample = {'time': utc(), 'monotonic': time.monotonic(), 'phase': SCENARIO['phase'], 'counts': dict(COUNTS), 'firing': firing, 'all_firing': sorted(a['labels']['alertname'] for a in alerts if a['state'] == 'firing')} SAMPLES.append(sample) except Exception as error: SAMPLES.append({'time': utc(), 'monotonic': time.monotonic(), 'phase': SCENARIO['phase'], 'monitor_error': type(error).__name__}) @@ -103,13 +103,15 @@ def main(): try: with ThreadPoolExecutor(max_workers=3) as pool: futures = [pool.submit(worker, actor, chat) for _ in range(3)] - phase('baseline'); time.sleep(60) - result['periods']['burst'] = incident('burst', 75) - time.sleep(100) - result['periods']['sustained'] = incident('sustained', 180) - time.sleep(100) - phase('complete') - STOP.set() + try: + phase('baseline'); time.sleep(60) + result['periods']['burst'] = incident('burst', 75) + time.sleep(100) + result['periods']['sustained'] = incident('sustained', 180) + time.sleep(100) + phase('complete') + finally: + STOP.set() for future in futures: future.result(timeout=15) finally: STOP.set(); monitor_thread.join(timeout=10) diff --git a/student-tests/slo_ready_incident.py b/student-tests/slo_ready_incident.py index c816668..f745af7 100644 --- a/student-tests/slo_ready_incident.py +++ b/student-tests/slo_ready_incident.py @@ -22,12 +22,14 @@ def prom(path, query=None): if query is not None: url += '?' + urllib.parse.urlencode({'query': query}) with urllib.request.urlopen(urllib.request.Request(url, headers={'Authorization': auth}), timeout=5) as r: return json.load(r)['data'] def metric(name): - result = prom('query', name + '{service_name="messenger-outbox"}')['result'] + result = prom('query', 'max(' + name + '{service_name="messenger-outbox"})')['result'] return float(result[0]['value'][1]) if result else None def sample(phase): alerts = prom('alerts')['alerts'] return {'time': utc(), 'monotonic': time.monotonic(), 'phase': phase, 'firing': sorted(a['labels']['alertname'] for a in alerts if a['state']=='firing' and a['labels']['alertname'].startswith('MessengerReadyBurn')), + 'all_firing': sorted(a['labels']['alertname'] for a in alerts if a['state']=='firing'), + 'queue_depth': metric('messenger_queue_depth'), 'queue_age': metric('messenger_queue_age_seconds'), 'eligible': metric('messenger_slo_ready_eligible_total'), 'bad': metric('messenger_slo_ready_bad_total')} def docker(*args): subprocess.run(['docker','compose',*args], cwd=ROOT, check=True, stdout=subprocess.DEVNULL) def run(): @@ -43,13 +45,15 @@ def run(): for _ in range(8): accepted.append(client.attachment(actor,chat)[0]) out['accepted_at']={'time':utc(),'monotonic':time.monotonic()} print('8 synthetic jobs accepted; waiting for burn alert', flush=True) + seen=set() deadline=time.monotonic()+100 while time.monotonic()=0 for value in out['counter_deltas'].values()),'counter reset' + out['load']=json.loads((ROOT/'evidence/review-tempo-load.json').read_text()) + assert out['load']['failed_requests']==0,out['load'] + out['result']='PASS' + finally: + if load is not None and load.poll() is None: + load.terminate();load.wait(timeout=15) + if stopped: docker('up','-d','--no-deps','tempo') + (ROOT/'evidence/review-tempo-outage.json').write_text(json.dumps(out,indent=2)+'\n') + print('PASS: Tempo outage; baseline/deltas/memory in evidence/review-tempo-outage.json') +if __name__=='__main__': main() diff --git a/student-tests/trace_retry.py b/student-tests/trace_retry.py index da858de..056143b 100644 --- a/student-tests/trace_retry.py +++ b/student-tests/trace_retry.py @@ -1,5 +1,7 @@ """Synthetic upload with a known W3C trace ID; wait through a worker restart.""" import secrets +import json +from datetime import datetime, timezone import sys import time from pathlib import Path @@ -18,3 +20,8 @@ print('trace_id:', trace_id, 'attachment_id:', attachment['id'], flush=True) started = time.monotonic() result = client.wait_ready(actor, attachment, wait_seconds=150) print('status:', result['status'], 'elapsed_seconds:', round(time.monotonic()-started, 3), flush=True) + +(Path(__file__).resolve().parents[1] / 'evidence/review-retry-job.json').write_text(json.dumps({ + 'utc': datetime.now(timezone.utc).isoformat(), 'trace_id': trace_id, + 'attachment_id': attachment['id'], 'status': result['status'], + 'elapsed_seconds': round(time.monotonic()-started, 3)}, indent=2)+'\n') diff --git a/uv.lock b/uv.lock index 359ef2c..33119d4 100644 --- a/uv.lock +++ b/uv.lock @@ -462,8 +462,8 @@ wheels = [ ] [[package]] -name = "mtusi2026lab4" -version = "0.1.0" +name = "mtusi2026lab6" +version = "1.6.0" source = { virtual = "." } dependencies = [ { name = "argon2-cffi" },