WIP: ЛР 6 — БВТ2301 — Маслов — на 5 #59
@@ -18,3 +18,20 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
**/__pycache__
|
||||
**/*.py[cod]
|
||||
.work
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
tests
|
||||
student-tests
|
||||
research
|
||||
scripts
|
||||
contracts
|
||||
*.md
|
||||
Makefile
|
||||
compose.yaml
|
||||
|
||||
.work*
|
||||
|
||||
student-unit
|
||||
|
||||
+14
-9
@@ -13,18 +13,23 @@ API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_BUCKET=messenger
|
||||
S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
|
||||
PUBLIC_PORT=8086
|
||||
API_CPUS=0.5
|
||||
API_MEMORY=768m
|
||||
DB_POOL_SIZE=10
|
||||
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_TIMEOUT_SECONDS=20
|
||||
JOB_LEASE_SECONDS=23
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
|
||||
OTEL_ENABLED=1
|
||||
GRAFANA_PORT=3006
|
||||
|
||||
@@ -1,18 +1,21 @@
|
||||
## Лабораторная 6
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Маслов, БВТ2301.
|
||||
Целевая оценка:5. Трек2 — SLO и обнаружение инцидентов.
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Собственная ЛР5: устойчивые jobs/outbox/relay/worker, четыре варианта, retry/replay/версии/cleanup. Добавлены OTel instrumentation API/relay/consumer/isolated child/DB/S3, persistent Prometheus/Loki/Tempo, Grafana provisioning18панелей, low-cardinality/privacy, конечные буферы Collector и multi-window burn-rate alerts.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Запуск: `uv sync --locked`, `make bootstrap`, `VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait`; API localhost8086/Grafana3006. Команды/определения/источники: [REPORT](../REPORT.md).
|
||||
|
||||
Реальные проверки:112HTTP/26unit; normal/retry causal traces и связанные logs; privacy canaries; реальные запросы Grafana. Полный SLO опыт1823.238с:6370подтверждённых сообщений и359файлов сохранены; краткие сбои0critical paging, warning API был; длинные инциденты firing/resolved измерены;90с отказ Loki/Tempo, queue≤64/RSS<384MiB и360успешных writes. Сводки/CSV/график в evidence. Проверка полного свежего clone ещё выполняется отдельно и не заявляется завершённой.
|
||||
|
||||
Ограничения: один host/одиночные stateful сервисы, учебные30mокна, цель99% намеренно нарушена контролируемыми сбоями; в конце опыта4неплановых503 сохранены в raw данных. Возможна потеря telemetry при отказе backend. HTTP только локальный; TLS/mTLS — ЛР7.
|
||||
|
||||
Источники: собственная ЛР5, задание, официальная документация OTel/Grafana/Prometheus. Использована помощь ИИ при проектировании, реализации, проверках и подготовке отчёта; чужие студенческие решения не использовались.
|
||||
|
||||
- [ ] Весь обязательный минимум подтверждён повторной полной регрессией из чистого клона.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; реальные receipts приложены.
|
||||
- [x] Отказы observability/SLO своего уровня воспроизведены.
|
||||
- [x] Секреты и приватные ключи вне Git; значения не включены в отчёт.
|
||||
- [x] README задания сохранён, REPORT описывает своё решение.
|
||||
|
||||
@@ -30,3 +30,8 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
.work/
|
||||
.work-*.txt
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
|
||||
COPY --from=uv /uv /usr/local/bin/uv
|
||||
WORKDIR /app
|
||||
ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --locked --no-dev --no-install-project
|
||||
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
|
||||
ARG VCS_REF=unknown
|
||||
LABEL org.opencontainers.image.revision=$VCS_REF
|
||||
WORKDIR /app
|
||||
ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080
|
||||
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY src /app/src
|
||||
COPY migrations /app/migrations
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"]
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "-m", "src"]
|
||||
@@ -28,3 +28,26 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
UV ?= uv
|
||||
BASE_URL = http://127.0.0.1:8086
|
||||
.PHONY: bootstrap up down lint test-student verify
|
||||
bootstrap:
|
||||
$(PYTHON) scripts/bootstrap.py
|
||||
up: bootstrap
|
||||
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait
|
||||
down:
|
||||
docker compose down
|
||||
lint:
|
||||
$(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py $(wildcard scripts/verify_*.py)
|
||||
$(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py $(wildcard scripts/verify_*.py)
|
||||
test-student:
|
||||
BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests
|
||||
verify:
|
||||
$(UV) run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json
|
||||
|
||||
.PHONY: unit reconcile
|
||||
unit:
|
||||
PYTHONPATH=. $(UV) run --locked pytest student-unit -q
|
||||
reconcile:
|
||||
docker compose run --rm --no-deps reconciler python -m src.reconcile
|
||||
|
||||
@@ -1,37 +1,92 @@
|
||||
# Отчёт — лабораторная 6
|
||||
# Лабораторная 6 — Маслов, БВТ2301
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
Цель — оценка 5, трек 2: SLO и обнаружение инцидентов. Собственный код ЛР5 перенесён поверх независимого skeleton ЛР6. Исходные README/COURSE/контракты/публичные tests и check/validate сохранены. Бизнес-API и устойчивый pipeline файлов продолжают работать; добавлена наблюдаемость всех стадий.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Воспроизведение
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Desktop/Compose v2, Git и uv0.13.7. Python3.14.7, SDK OpenTelemetry1.45.2, Collector0.165.0, Prometheus3.16.0, Loki3.7.10, Tempo3.1.1, Grafana13.3.1. Версии и image digests закреплены. MinIO/mc собираются из точных upstream commits с GoSumDB.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
uv sync --locked
|
||||
make bootstrap
|
||||
VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait
|
||||
make check validate lint unit PYTHON=.venv/bin/python
|
||||
make test BASE_URL=http://127.0.0.1:8086 PYTHON=.venv/bin/python
|
||||
make test-student
|
||||
uv run --locked python -m scripts.verify_telemetry --output evidence/telemetry-repeat.json
|
||||
uv run --locked python -m scripts.verify_observability_persistence --causal-evidence evidence/telemetry-repeat.json --output evidence/persistence-repeat.json
|
||||
uv run --locked python -m scripts.research_slo --output evidence/slo-repeat.json
|
||||
uv run --locked python -m research.analyze_slo evidence/slo-repeat.json --output evidence/slo-analysis-repeat
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
API: http://127.0.0.1:8086. Grafana: http://127.0.0.1:3006, admin, пароль в локальном `secrets/grafana_password`. Bootstrap сохраняет прежние файлы и не печатает значения. Provisioning трёх datasources и dashboard выполняется автоматически. Внутренние OTLP/metrics/backend ports на host не публикуются. `docker compose down` сохраняет данные; secrets и volumes не входят в Git. Fault hooks включаются только явным `compose.test.yaml`.
|
||||
|
||||
## Доказательства критериев
|
||||
Проверка чистого клона: `uv run --locked python scripts/verify_clean.py --output evidence/clean-clone-repeat.json`. Она создаёт собственные временные clone/venv/secrets/volumes/ports/image, выполняет полную регрессию и удаляет только временный проект. Основной стенд перед ней следует остановить, сохранив тома. Имена output должны быть новыми: предыдущие результаты не перезаписываются.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
## Архитектура и сигналы
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
API/relay/worker/observer → OTLP/HTTP → Collector → Prometheus/Loki/Tempo → запросы Grafana. Grafana показывает данные; сама она не заменяет хранилища трёх сигналов. Каждый backend и Grafana имеют отдельный persistent volume. API-реплики, PG, S3, broker, worker, сессии, ACL и идемпотентность сохранены.
|
||||
|
||||
## Эксперименты и отказы
|
||||
Migration005 сохраняет W3C context в принимающей транзакции вместе с jobs/outbox. Relay создаёт producer span и передаёт его контекст в stream. Worker создаёт consumer span, передаёт carrier изолированному дочернему процессу; в trace видны обработка изображения, S3 и PostgreSQL. Повторная попытка сохраняет причинную связь. `traceparent` сохраняется при отбрасывании слишком длинного `tracestate`, чтобы не нарушать строгую схему jobs.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Логи структурные: event/service/trace/span/request/технический job ID. Экспорт полей задан allowlist. Тела сообщений, изображения, пароли, Authorization, SQL-тексты, raw URL и exception message/stack не экспортируются. Идентификаторы пользователей/чатов/сообщений не становятся labels. Метрики используют route template, ограниченный HTTP method/status class, job type/outcome, service instance/version.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Dashboard `messenger-lab6` содержит 18 панелей: RPS, 5xx, HTTP p50/p95/p99, jobs/duration, время до ready, очередь/возраст, worker busy/capacity, DB pool used/waiting, память/CPU и Collector. Percentile в dashboard вычисляется `histogram_quantile` по сумме bucket rates с сохранённым le; значения разных реплик сначала объединяются. Определения и единицы находятся в импортируемом JSON. HTTP SDK duration измеряет ASGI response; end-to-end клиентские quantiles в исследовании — отдельные величины, вычисленные nearest rank по успешным HTTP запросам.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
SDK traces/logs: очередь 512 записей, batch64, конечные таймауты. Collector: memory limiter192MiB/spike32, batch≤256, очереди logs/traces по32 пакета, один отправитель, конечные повторы до10с, nonblocking overflow. Collector CPU0.5/RAM384MiB, Tempo CPU0.5/RAM768MiB. Потери telemetry при длительном отказе допустимы и наблюдаемы; не заявляется надёжная доставка каждого span.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
## SLI/SLO и гипотеза
|
||||
|
||||
Две учебные цели 99% за окно30мин: ответы business ingress без5xx и готовность принятого файла не позднее10с. API SLI берётся с HAProxy business backend, включая ingress503; health использует отдельный backend. Ready SLI сравнивает PG created_at принимающей транзакции с committed_at публикации результата. Начальная граница консервативно включает завершение принимающей транзакции.
|
||||
|
||||
Незавершённые после10с и failed учитываются как bad, а не исчезают из знаменателя. Решение сохраняется однократно в PG; replay не стирает прежний bad, obsolete до решения исключён. Histogram времени готовности содержит только завершения и не заменяет этот SLI.
|
||||
|
||||
Гипотеза: сочетание короткого и длинного окна в burn-rate rule отсечёт краткий всплеск от критического оповещения и обнаружит длительное нарушение обоих SLO. Burn rate — доля плохих событий/0.01. Fast: >14.4 одновременно в30с и2мин, for45с. Slow: >6 в2мин и5мин, for60с. Дополнительно warning ApiHighErrorRate и QueueLagGrowing. Это ускоренный учебный профиль; значения не выдаются за универсальную production policy.
|
||||
|
||||
## Методика и результаты исследования
|
||||
|
||||
Полный успешный опыт — `evidence/slo-research-verified.json`, CSV запросов и `evidence/slo-analysis-verified/summary.json`, таблица фаз и [график](evidence/slo-analysis-verified/slo-timeline.png). Длительность1823.238с. Нагрузка: открытое планирование4 message POST/с, максимум16 in-flight; PNG320×200 каждые5с. Docker VM:8CPU и фактический объём памяти в receipt; все service quotas сохранены там же.
|
||||
|
||||
Опыт проводился при API0.5CPU/768MiB на реплику (точные limits в receipt). Финальный default API увеличен до1CPU после отдельной проверки login latency; это не тот же ресурсный профиль и результаты не выдаются за повторное измерение на1CPU. Для повторения именно опыта сначала развернуть `COMPOSE_FILE=compose.yaml:compose.test.yaml API_CPUS=0.5 VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait`, затем запускать research с теми же COMPOSE_FILE/API_CPUS.
|
||||
|
||||
После разогрева300с воспроизводятся короткие и длительные сбои. Для API — задержка SQL INSERT только синтетического чата, превышающая statement_timeout. Для ready — остановка worker. Затем оба telemetry хранилища Loki/Tempo останавливаются на90с, при этом сообщения и задания продолжают приниматься. После восстановления найдены свежие trace и log. В конце прекращается нагрузка и через публичный API проверяются все подтверждённые данные.
|
||||
|
||||
| Длительный инцидент / оповещение | Обнаружение, с после действия | Снятие, с после восстановления компонента |
|
||||
|---|---:|---:|
|
||||
| API / FastBurn |73.368|39.261|
|
||||
| API / SlowBurn |84.651|127.152|
|
||||
| API / HighErrorRate warning |44.859|127.152|
|
||||
| Worker / QueueLag warning |47.281|181.174|
|
||||
| Worker / FastBurn |76.710|235.563|
|
||||
| Worker / SlowBurn |85.981|303.950|
|
||||
|
||||
При коротких сбоях критических paging событий0. Короткий API сбой всё же вызвал warning ApiHighErrorRate во время восстановления: не утверждается отсутствие всех оповещений. Очередь и окна не очищаются мгновенно после запуска worker; время снятия включает дренирование и исчезновение старых нарушений из окон.
|
||||
|
||||
В ходе90с отказа telemetry сохранены360 успешных message POST, Collector queue достигла64 при суммарной ёмкости64. Его RSS в опыте238.918–239.035MiB при лимите384MiB. После восстановления очередь0, новая trace/log доступны. Счётчик неуспешных отправок показывает попытки, а не уникально потерянные spans; точное количество потерянных spans не выводится из него.
|
||||
|
||||
В конце доступны все6370 подтверждённых сообщений, все359 принятых файлов находятся в ready; проверенные исходники побайтно совпадают. Это проверка сохранности подтверждённых операций, а не утверждение, что все запросы были успешны. Во время преднамеренных сбоев SLI API89.925%, ready79.832%; цель99% специально нарушена, что и должны обнаружить правила. В заключительной фазе также зарегистрированы4 ответа503; они не скрываются из CSV или расчётов.
|
||||
|
||||
Частота scrape/evaluation5с и экспорт ограничивают разрешение, а реальный polling наблюдателя дал промежутки5.111–38.223с, медиана9.663с. Три десятичных знака — арифметика временных меток, не миллисекундная точность обнаружения. Клиентские фазовые сводки используют start timestamp внутри фактически измеренного hold; промежутки orchestration исключены, raw CSV сохранён.
|
||||
|
||||
## Проверка и разбор инцидентов
|
||||
|
||||
Реальные результаты до clean clone:112 HTTP tests PASS416.52с (`http-first.json`),26 unit, smoke/offline/full OpenAPI/lint. `telemetry-first.json` подтверждает цепочку normal upload25spans/1attempt и retry33spans/2attempts; реальные parent IDs API→producer→consumer→child→DB/S3, связанные Loki trace/span/request IDs, privacy canaries,1131 ограниченных metric series,18панелей/3 datasource health и реальные Grafana query frames.
|
||||
|
||||
Для API инцидента сохранена ошибочная трасса с db.postgresql ≥2с и ERROR, при этом очередь не объясняет деградацию. Для worker инцидента upload202 продолжает работать, возраст и очередь растут, worker spans прекращаются; после запуска для того же attachment появляется consumer/processing chain и ready. Диагностировать следует по этим наблюдениям, а не одному коду500. Пошаговые действия и точные запросы: [RUNBOOK](infra/otel/RUNBOOK.md).
|
||||
|
||||
Предварительные неуспешные опыты сохранены с complete=false. Они выявили имя actual RSS metric, недостаточный запас памяти, слишком раннее фиксированное ожидание resolved и строковый тип Compose mem_limit. Последнее исправлено нормализацией чисел. Ни один неполный опыт не выдан за успешный30минутный.
|
||||
|
||||
Первый fresh clone (`clean-clone-final-failed.json`, source2c7b89b) прошёл static checks/26unit, но остановился на unhealthy одной API-реплики до HTTP suite. Его результат сохранён как INCOMPLETE. Выявлено несовпадение default API RAM384MiB с фактически исследованными768MiB; default приведён к исследованному профилю, задан конечный startup health budget и добавлены progress/санитизированная диагностика до удаления временного стенда. Причина первого неуспешного health без сохранённых container logs окончательно не установлена. Повторный fresh clone sourcea19e805 прошёл112HTTP за558.13с и реальные telemetry/persistence проверки; затем публичный resilience test поймал login2.12489с при своём timeout2с. Неуспешный receipt `clean-clone-final-failed-af292f62.json` сохранён. Default API повышен с0.5до1CPU; параметры Argon2 и публичный тест не менялись. Целевая проверка `cpu-budget-public.json` дала10из10логинов при timeout2с, максимум0.66730с, и PASS исходных public failover/persistence; recovery9.174с≤10с, сообщения/файл/прежняя сессия сохранены. Новый полный fresh clone с telemetry persistence/cluster/pipeline/cleanup выполняется отдельно; до его успешного receipt воспроизводимость из чистого клона не заявляется.
|
||||
|
||||
## Ответы на контрольные вопросы
|
||||
|
||||
1. Метрика показывает масштаб/время проблемы: например доля5xx или возраст очереди. Лог — конкретное событие и технические ID. Trace — путь одного запроса/задания и длительности отдельных стадий. Вместе они позволяют перейти от общей деградации к причине.
|
||||
2. Каждый новый chat_id создаёт отдельный временной ряд для каждой комбинации labels. Рост чатов умножает число рядов и стоимость RAM/индекса/запросов. Поэтому chat_id оставляют в техническом логе/trace, а метрику группируют по ограниченным признакам.
|
||||
3. Контекст traceparent/tracestate сохраняется вместе с заданием, relay создаёт producer span и записывает его carrier в уведомление. Worker извлекает его и создаёт consumer span с правильным parent; передача в child не теряется при смене процесса. Durable outbox отделяет сохранение задания от доставки уведомления.
|
||||
4. Среднее скрывает редкие долгие ответы; p99 описывает хвост. Усреднять p99 реплик нельзя: разные объёмы запросов и распределения не восстанавливаются из двух quantiles. Нужно сложить совместимые histogram buckets реплик и вычислить общий percentile.
|
||||
|
||||
## Источники и ограничения
|
||||
|
||||
Задание, собственная ЛР5 и официальные материалы: [OTLP Python](https://opentelemetry.io/docs/languages/python/exporters/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [Tempo3.1.1 config](https://github.com/grafana/tempo/blob/v3.1.1/example/docker-compose/single-binary/tempo.yaml), [Collector](https://opentelemetry.io/docs/collector/configuration/), [Prometheus alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/). При проектировании, реализации, проверках и подготовке отчёта использована помощь ИИ. Чужие студенческие решения/истории/измерения не использовались.
|
||||
|
||||
Один host и одиночные PG/S3/broker/LB остаются SPOF; нет защиты от потери диска. SLO исследован один раз полностью на данном профиле; статистическая обобщаемость не заявляется. Остались явные потери запросов/telemetry при искусственных отказах; они отражены в raw receipts. HTTP используется только локально с синтетическими данными; TLS/mTLS вводятся независимо в ЛР7.
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
# Explicit test-only fault instrumentation, never enabled by default.
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
PIPELINE_FAULTS: "1"
|
||||
relay:
|
||||
environment:
|
||||
PIPELINE_FAULTS: "1"
|
||||
worker:
|
||||
environment:
|
||||
PIPELINE_FAULTS: "1"
|
||||
JOB_TIMEOUT_SECONDS: "12"
|
||||
JOB_LEASE_SECONDS: "15"
|
||||
controller:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
pull_policy: never
|
||||
command: [python, -c, "import time; time.sleep(86400)"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
secrets: [database_url, broker_url, s3_access_key, s3_secret_key]
|
||||
cpus: 0.5
|
||||
mem_limit: 192m
|
||||
memswap_limit: 192m
|
||||
read_only: true
|
||||
healthcheck: {disable: true}
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
+508
@@ -0,0 +1,508 @@
|
||||
name: mtusi-lab6-maslov
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- pg_isready -U messenger -d messenger
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 1
|
||||
mem_limit: 512m
|
||||
memswap_limit: 512m
|
||||
cache:
|
||||
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
|
||||
command:
|
||||
- valkey-server
|
||||
- /run/secrets/valkey.conf
|
||||
secrets:
|
||||
- cache_password
|
||||
- source: valkey_config
|
||||
target: valkey.conf
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- REDISCLI_AUTH="$$(cat /run/secrets/cache_password)" valkey-cli ping
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
migrate:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
pull_policy: never
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- src.migrate
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets:
|
||||
- database_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
read_only: true
|
||||
api:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
VCS_REF: ${VCS_REF:-unknown}
|
||||
scale: ${API_REPLICAS:-2}
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DB_POOL_SIZE: ${DB_POOL_SIZE:-10}
|
||||
OTEL_ENABLED: ${OTEL_ENABLED:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
|
||||
APP_VERSION: ${VCS_REF:-development}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- s3_access_key
|
||||
- s3_secret_key
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
cache:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
cpus: ${API_CPUS:-1}
|
||||
mem_limit: ${API_MEMORY:-768m}
|
||||
memswap_limit: ${API_MEMORY:-768m}
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).read()"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
start_period: 30s
|
||||
retries: 12
|
||||
pids_limit: 128
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=128m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
stop_grace_period: 10s
|
||||
s3:
|
||||
image: ${S3_IMAGE:-mtusi-storage:minio-20251015}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/s3.Dockerfile
|
||||
target: server
|
||||
secrets:
|
||||
- s3_root_user
|
||||
- s3_root_password
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
cpus: 1
|
||||
mem_limit: 1024m
|
||||
memswap_limit: 1024m
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,nosuid,size=32m
|
||||
environment:
|
||||
HOME: /tmp
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
s3-bootstrap:
|
||||
image: ${MC_IMAGE:-mtusi-storage:mc-20250813}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/s3.Dockerfile
|
||||
target: bootstrap
|
||||
secrets:
|
||||
- s3_root_user
|
||||
- s3_root_password
|
||||
- s3_access_key
|
||||
- s3_secret_key
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,nosuid,size=16m
|
||||
reconciler:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
pull_policy: never
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- src.reconcile
|
||||
- --apply
|
||||
- --loop
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key
|
||||
- s3_secret_key
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
cpus: 0.25
|
||||
mem_limit: 256m
|
||||
memswap_limit: 256m
|
||||
read_only: true
|
||||
healthcheck:
|
||||
disable: true
|
||||
broker:
|
||||
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
|
||||
command:
|
||||
- valkey-server
|
||||
- /run/secrets/broker.conf
|
||||
secrets:
|
||||
- broker_password
|
||||
- source: broker_config
|
||||
target: broker.conf
|
||||
volumes:
|
||||
- broker_data:/data
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
memswap_limit: 256m
|
||||
relay:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
pull_policy: never
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- src.pipeline
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
OTEL_ENABLED: ${OTEL_ENABLED:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
|
||||
APP_VERSION: ${VCS_REF:-development}
|
||||
secrets:
|
||||
- database_url
|
||||
- broker_url
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
cpus: 0.25
|
||||
mem_limit: 192m
|
||||
memswap_limit: 192m
|
||||
read_only: true
|
||||
healthcheck:
|
||||
disable: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
worker:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
pull_policy: never
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- src.worker
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-23}
|
||||
OTEL_ENABLED: ${OTEL_ENABLED:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
|
||||
APP_VERSION: ${VCS_REF:-development}
|
||||
secrets:
|
||||
- database_url
|
||||
- broker_url
|
||||
- s3_access_key
|
||||
- s3_secret_key
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
cpus: 2
|
||||
mem_limit: 1536m
|
||||
memswap_limit: 1536m
|
||||
pids_limit: 128
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=64m
|
||||
healthcheck:
|
||||
disable: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
stop_grace_period: 25s
|
||||
lb:
|
||||
image: haproxy:3.2.1-alpine@sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9
|
||||
ports:
|
||||
- 127.0.0.1:${PUBLIC_PORT:-8086}:8080
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,nosuid,size=4m
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
stop_grace_period: 10s
|
||||
observer:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
pull_policy: never
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- src.observer
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
OTEL_ENABLED: ${OTEL_ENABLED:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
|
||||
APP_VERSION: ${VCS_REF:-development}
|
||||
secrets:
|
||||
- database_url
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
cpus: 0.25
|
||||
mem_limit: 192m
|
||||
memswap_limit: 192m
|
||||
read_only: true
|
||||
healthcheck:
|
||||
disable: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
otel:
|
||||
image: otel/opentelemetry-collector-contrib:0.165.0@sha256:e40d61f10c9a617cca1ffd1be4d7e2b3a4b0f0001798ff07e58cabe82e618cea
|
||||
user: 10001:10001
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otel/config.yaml:ro
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
memswap_limit: 384m
|
||||
environment:
|
||||
GOMEMLIMIT: 224MiB
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=32m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 128
|
||||
command:
|
||||
- --config=/etc/otel/config.yaml
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.16.0@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06
|
||||
user: 65534:65534
|
||||
volumes:
|
||||
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
|
||||
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
memswap_limit: 384m
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=32m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 128
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yaml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=24h
|
||||
- --storage.tsdb.retention.size=256MB
|
||||
depends_on:
|
||||
telemetry-init:
|
||||
condition: service_completed_successfully
|
||||
loki:
|
||||
image: grafana/loki:3.7.10@sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7
|
||||
user: 10001:10001
|
||||
volumes:
|
||||
- ./infra/otel/loki.yaml:/etc/loki/config.yaml:ro
|
||||
- loki_data:/loki
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
memswap_limit: 384m
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=32m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 128
|
||||
command:
|
||||
- -config.file=/etc/loki/config.yaml
|
||||
depends_on:
|
||||
telemetry-init:
|
||||
condition: service_completed_successfully
|
||||
tempo:
|
||||
image: grafana/tempo:3.1.1@sha256:64e2fbd56d9c2b3412a4b430a58ea59e91a670b1e75f920a0c702213ad145044
|
||||
user: 10001:10001
|
||||
volumes:
|
||||
- ./infra/otel/tempo.yaml:/etc/tempo/config.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
cpus: 0.5
|
||||
mem_limit: 768m
|
||||
memswap_limit: 768m
|
||||
environment:
|
||||
GOMEMLIMIT: 384MiB
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=32m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 128
|
||||
command:
|
||||
- -config.file=/etc/tempo/config.yaml
|
||||
depends_on:
|
||||
telemetry-init:
|
||||
condition: service_completed_successfully
|
||||
grafana:
|
||||
image: grafana/grafana:13.3.1@sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf
|
||||
user: 472:472
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
memswap_limit: 384m
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=32m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 128
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
|
||||
GF_AUTH_ANONYMOUS_ENABLED: 'false'
|
||||
GF_USERS_ALLOW_SIGN_UP: 'false'
|
||||
GF_ANALYTICS_REPORTING_ENABLED: 'false'
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
|
||||
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: 'false'
|
||||
GF_PLUGINS_PREINSTALL_DISABLED: 'true'
|
||||
GF_LOG_LEVEL: warn
|
||||
GOMEMLIMIT: 256MiB
|
||||
secrets:
|
||||
- grafana_password
|
||||
ports:
|
||||
- 127.0.0.1:${GRAFANA_PORT:-3006}:3000
|
||||
depends_on:
|
||||
telemetry-init:
|
||||
condition: service_completed_successfully
|
||||
telemetry-init:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
pull_policy: never
|
||||
user: 0:0
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- import os; [(os.makedirs(p,exist_ok=True),os.chown(p,u,u)) for p,u in [('/data/prometheus',65534),('/data/loki',10001),('/data/tempo',10001),('/data/grafana',472)]]
|
||||
volumes:
|
||||
- prometheus_data:/data/prometheus
|
||||
- loki_data:/data/loki
|
||||
- tempo_data:/data/tempo
|
||||
- grafana_data:/data/grafana
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
cap_add:
|
||||
- CHOWN
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cpus: 0.1
|
||||
mem_limit: 64m
|
||||
healthcheck:
|
||||
disable: true
|
||||
secrets:
|
||||
broker_password:
|
||||
file: ./secrets/broker_password
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
broker_config:
|
||||
file: ./secrets/broker.conf
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_access_key:
|
||||
file: ./secrets/s3_access_key
|
||||
s3_secret_key:
|
||||
file: ./secrets/s3_secret_key
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_password:
|
||||
file: ./secrets/cache_password
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
valkey_config:
|
||||
file: ./secrets/valkey.conf
|
||||
name: valkey.conf
|
||||
grafana_password:
|
||||
file: ./secrets/grafana_password
|
||||
volumes:
|
||||
broker_data: null
|
||||
s3_data: null
|
||||
postgres_data: null
|
||||
prometheus_data: null
|
||||
loki_data: null
|
||||
tempo_data: null
|
||||
grafana_data: null
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"date": "2026-11-20T15:15:34.500702+00:00",
|
||||
"source_commit": "2c7b89bb8e68ac79971b3314a8f7f97eb30f0cf5",
|
||||
"fresh_secrets_environment_and_volumes": true,
|
||||
"command": "python scripts/verify_clean.py",
|
||||
"fixed_host_port_for_restart": 57033,
|
||||
"static_checks": ".venv/bin/python scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n.venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\n<home>/.local/bin/uv run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py scripts/verify_clean.py scripts/verify_cleanup.py scripts/verify_cluster.py scripts/verify_observability_persistence.py scripts/verify_pipeline.py scripts/verify_telemetry.py\n46 files already formatted\n<home>/.local/bin/uv run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py scripts/verify_clean.py scripts/verify_cleanup.py scripts/verify_cluster.py scripts/verify_observability_persistence.py scripts/verify_pipeline.py scripts/verify_telemetry.py\nAll checks passed!",
|
||||
"unit_tests": "PYTHONPATH=. <home>/.local/bin/uv run --locked pytest student-unit -q\n.......................... [100%]\n26 passed in 10.87s",
|
||||
"failed_action": "docker compose up -d --build --wait",
|
||||
"result": "INCOMPLETE"
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"source_commit": "a19e8053a7b22f067c2ac7a60956adecacc275a2",
|
||||
"api_cpus": 1,
|
||||
"client_timeout_seconds": 2,
|
||||
"complete": true,
|
||||
"login_trials": [
|
||||
{
|
||||
"register_seconds": 0.5860111659858376,
|
||||
"login_seconds": 0.5677100410684943,
|
||||
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
|
||||
},
|
||||
{
|
||||
"register_seconds": 0.5128639999311417,
|
||||
"login_seconds": 0.3723337920382619,
|
||||
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
|
||||
},
|
||||
{
|
||||
"register_seconds": 0.35827833297662437,
|
||||
"login_seconds": 0.6672967500053346,
|
||||
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
|
||||
},
|
||||
{
|
||||
"register_seconds": 0.4009002910461277,
|
||||
"login_seconds": 0.577231707982719,
|
||||
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
|
||||
},
|
||||
{
|
||||
"register_seconds": 0.346562874969095,
|
||||
"login_seconds": 0.2660798749420792,
|
||||
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
|
||||
},
|
||||
{
|
||||
"register_seconds": 0.4216097079915926,
|
||||
"login_seconds": 0.3548308340832591,
|
||||
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
|
||||
},
|
||||
{
|
||||
"register_seconds": 0.3752310830168426,
|
||||
"login_seconds": 0.45202583307400346,
|
||||
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
|
||||
},
|
||||
{
|
||||
"register_seconds": 0.5370608329540119,
|
||||
"login_seconds": 0.34091954200994223,
|
||||
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
|
||||
},
|
||||
{
|
||||
"register_seconds": 0.35265858308412135,
|
||||
"login_seconds": 0.46283045900054276,
|
||||
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
|
||||
},
|
||||
{
|
||||
"register_seconds": 0.44550816691480577,
|
||||
"login_seconds": 0.4593870830722153,
|
||||
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
|
||||
}
|
||||
],
|
||||
"failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 82, \"failed_attempts\": 6, \"first_recovery_seconds\": 9.174}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
|
||||
"persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"source_commit": "8f3ff33b2dcb1cf3082535acf99ee4045a80400d",
|
||||
"command": "BASE_URL=http://127.0.0.1:8086 uv run --locked pytest student-tests -q",
|
||||
"elapsed_seconds": 417.533,
|
||||
"exit_code": 0,
|
||||
"summary": "112 passed in 416.52s (0:06:56)",
|
||||
"result": "PASS"
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
phase,seconds,started_requests,statuses,successful_rps,success_p50_ms,success_p95_ms,success_p99_ms,critical_firing_names,warning_firing_names
|
||||
warmup,300.238,1200,{'201': 1200},3.997,18.907,154.932,396.121,[],[]
|
||||
api_brief,8.531,34,"{'503': 32, '-1': 2}",0.0,,,,[],[]
|
||||
api_brief_recovery,120.002,480,"{'201': 479, '0': 1}",3.992,27.665,229.005,791.96,[],['ApiHighErrorRate']
|
||||
api_sustained,185.636,743,{'503': 743},0.0,,,,"['ApiFastBurn', 'ApiSlowBurn']",['ApiHighErrorRate']
|
||||
api_recovery,156.766,627,{'201': 627},4.0,23.464,250.67,1807.065,"['ApiFastBurn', 'ApiSlowBurn']",['ApiHighErrorRate']
|
||||
ready_brief,5.004,20,{'201': 20},3.997,18.589,41.825,59.7,[],[]
|
||||
ready_brief_recovery,59.986,240,{'201': 240},4.001,29.034,194.65,396.574,[],[]
|
||||
ready_sustained,152.489,610,{'201': 610},4.0,21.872,150.518,429.458,"['ReadyFastBurn', 'ReadySlowBurn']",['QueueLagGrowing']
|
||||
ready_recovery,303.953,1216,"{'201': 1215, '0': 1}",3.997,31.974,256.132,866.055,"['ReadyFastBurn', 'ReadySlowBurn']",['QueueLagGrowing']
|
||||
backend_unavailable,90.001,360,{'201': 360},4.0,21.961,123.103,221.753,[],[]
|
||||
backend_recovery,59.999,240,{'201': 240},4.0,27.369,154.043,469.609,[],[]
|
||||
steady_after_recovery,268.806,1075,"{'201': 1071, '503': 4}",3.984,27.293,514.282,1556.851,[],[]
|
||||
|
Binary file not shown.
|
After Width: | Height: | Size: 187 KiB |
@@ -0,0 +1,404 @@
|
||||
{
|
||||
"source_commit": "7282aa8b18cc33cf243cc8d82ff2fdd71079766a",
|
||||
"runtime_versions": [
|
||||
"752a5e6c0d33c2856f7ac8e66074e4adfb9a699d"
|
||||
],
|
||||
"elapsed_seconds": 1823.238,
|
||||
"sample_count": 172,
|
||||
"sample_gap_seconds": {
|
||||
"min": 5.110785245895386,
|
||||
"median": 9.663292169570923,
|
||||
"max": 38.223085165023804
|
||||
},
|
||||
"client_quantile_method": "nearest rank of successful end-to-end writes; not histogram interpolation",
|
||||
"phase_assignment": "request start timestamp inside measured hold; orchestration gaps excluded",
|
||||
"phases": [
|
||||
{
|
||||
"phase": "warmup",
|
||||
"seconds": 300.238,
|
||||
"started_requests": 1200,
|
||||
"statuses": {
|
||||
"201": 1200
|
||||
},
|
||||
"successful_rps": 3.997,
|
||||
"success_p50_ms": 18.907,
|
||||
"success_p95_ms": 154.932,
|
||||
"success_p99_ms": 396.121,
|
||||
"critical_firing_names": [],
|
||||
"warning_firing_names": []
|
||||
},
|
||||
{
|
||||
"phase": "api_brief",
|
||||
"seconds": 8.531,
|
||||
"started_requests": 34,
|
||||
"statuses": {
|
||||
"503": 32,
|
||||
"-1": 2
|
||||
},
|
||||
"successful_rps": 0.0,
|
||||
"success_p50_ms": null,
|
||||
"success_p95_ms": null,
|
||||
"success_p99_ms": null,
|
||||
"critical_firing_names": [],
|
||||
"warning_firing_names": []
|
||||
},
|
||||
{
|
||||
"phase": "api_brief_recovery",
|
||||
"seconds": 120.002,
|
||||
"started_requests": 480,
|
||||
"statuses": {
|
||||
"201": 479,
|
||||
"0": 1
|
||||
},
|
||||
"successful_rps": 3.992,
|
||||
"success_p50_ms": 27.665,
|
||||
"success_p95_ms": 229.005,
|
||||
"success_p99_ms": 791.96,
|
||||
"critical_firing_names": [],
|
||||
"warning_firing_names": [
|
||||
"ApiHighErrorRate"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "api_sustained",
|
||||
"seconds": 185.636,
|
||||
"started_requests": 743,
|
||||
"statuses": {
|
||||
"503": 743
|
||||
},
|
||||
"successful_rps": 0.0,
|
||||
"success_p50_ms": null,
|
||||
"success_p95_ms": null,
|
||||
"success_p99_ms": null,
|
||||
"critical_firing_names": [
|
||||
"ApiFastBurn",
|
||||
"ApiSlowBurn"
|
||||
],
|
||||
"warning_firing_names": [
|
||||
"ApiHighErrorRate"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "api_recovery",
|
||||
"seconds": 156.766,
|
||||
"started_requests": 627,
|
||||
"statuses": {
|
||||
"201": 627
|
||||
},
|
||||
"successful_rps": 4.0,
|
||||
"success_p50_ms": 23.464,
|
||||
"success_p95_ms": 250.67,
|
||||
"success_p99_ms": 1807.065,
|
||||
"critical_firing_names": [
|
||||
"ApiFastBurn",
|
||||
"ApiSlowBurn"
|
||||
],
|
||||
"warning_firing_names": [
|
||||
"ApiHighErrorRate"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "ready_brief",
|
||||
"seconds": 5.004,
|
||||
"started_requests": 20,
|
||||
"statuses": {
|
||||
"201": 20
|
||||
},
|
||||
"successful_rps": 3.997,
|
||||
"success_p50_ms": 18.589,
|
||||
"success_p95_ms": 41.825,
|
||||
"success_p99_ms": 59.7,
|
||||
"critical_firing_names": [],
|
||||
"warning_firing_names": []
|
||||
},
|
||||
{
|
||||
"phase": "ready_brief_recovery",
|
||||
"seconds": 59.986,
|
||||
"started_requests": 240,
|
||||
"statuses": {
|
||||
"201": 240
|
||||
},
|
||||
"successful_rps": 4.001,
|
||||
"success_p50_ms": 29.034,
|
||||
"success_p95_ms": 194.65,
|
||||
"success_p99_ms": 396.574,
|
||||
"critical_firing_names": [],
|
||||
"warning_firing_names": []
|
||||
},
|
||||
{
|
||||
"phase": "ready_sustained",
|
||||
"seconds": 152.489,
|
||||
"started_requests": 610,
|
||||
"statuses": {
|
||||
"201": 610
|
||||
},
|
||||
"successful_rps": 4.0,
|
||||
"success_p50_ms": 21.872,
|
||||
"success_p95_ms": 150.518,
|
||||
"success_p99_ms": 429.458,
|
||||
"critical_firing_names": [
|
||||
"ReadyFastBurn",
|
||||
"ReadySlowBurn"
|
||||
],
|
||||
"warning_firing_names": [
|
||||
"QueueLagGrowing"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "ready_recovery",
|
||||
"seconds": 303.953,
|
||||
"started_requests": 1216,
|
||||
"statuses": {
|
||||
"201": 1215,
|
||||
"0": 1
|
||||
},
|
||||
"successful_rps": 3.997,
|
||||
"success_p50_ms": 31.974,
|
||||
"success_p95_ms": 256.132,
|
||||
"success_p99_ms": 866.055,
|
||||
"critical_firing_names": [
|
||||
"ReadyFastBurn",
|
||||
"ReadySlowBurn"
|
||||
],
|
||||
"warning_firing_names": [
|
||||
"QueueLagGrowing"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "backend_unavailable",
|
||||
"seconds": 90.001,
|
||||
"started_requests": 360,
|
||||
"statuses": {
|
||||
"201": 360
|
||||
},
|
||||
"successful_rps": 4.0,
|
||||
"success_p50_ms": 21.961,
|
||||
"success_p95_ms": 123.103,
|
||||
"success_p99_ms": 221.753,
|
||||
"critical_firing_names": [],
|
||||
"warning_firing_names": []
|
||||
},
|
||||
{
|
||||
"phase": "backend_recovery",
|
||||
"seconds": 59.999,
|
||||
"started_requests": 240,
|
||||
"statuses": {
|
||||
"201": 240
|
||||
},
|
||||
"successful_rps": 4.0,
|
||||
"success_p50_ms": 27.369,
|
||||
"success_p95_ms": 154.043,
|
||||
"success_p99_ms": 469.609,
|
||||
"critical_firing_names": [],
|
||||
"warning_firing_names": []
|
||||
},
|
||||
{
|
||||
"phase": "steady_after_recovery",
|
||||
"seconds": 268.806,
|
||||
"started_requests": 1075,
|
||||
"statuses": {
|
||||
"201": 1071,
|
||||
"503": 4
|
||||
},
|
||||
"successful_rps": 3.984,
|
||||
"success_p50_ms": 27.293,
|
||||
"success_p95_ms": 514.282,
|
||||
"success_p99_ms": 1556.851,
|
||||
"critical_firing_names": [],
|
||||
"warning_firing_names": []
|
||||
}
|
||||
],
|
||||
"incidents": [
|
||||
{
|
||||
"signal": "api",
|
||||
"action_started": 1795185997.2563121,
|
||||
"action_finished": 1795185997.581769,
|
||||
"restore_started": 1795186183.218286,
|
||||
"restore_finished": 1795186185.962821,
|
||||
"timings": {
|
||||
"ApiFastBurn": {
|
||||
"detect_seconds": 73.368,
|
||||
"resolve_seconds": 39.261
|
||||
},
|
||||
"ApiHighErrorRate": {
|
||||
"detect_seconds": 44.859,
|
||||
"resolve_seconds": 127.152
|
||||
},
|
||||
"ApiSlowBurn": {
|
||||
"detect_seconds": 84.651,
|
||||
"resolve_seconds": 127.152
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"signal": "ready",
|
||||
"action_started": 1795186414.030848,
|
||||
"action_finished": 1795186415.9932559,
|
||||
"restore_started": 1795186568.482792,
|
||||
"restore_finished": 1795186569.2213361,
|
||||
"timings": {
|
||||
"QueueLagGrowing": {
|
||||
"detect_seconds": 47.281,
|
||||
"resolve_seconds": 181.174
|
||||
},
|
||||
"ReadyFastBurn": {
|
||||
"detect_seconds": 76.71,
|
||||
"resolve_seconds": 235.563
|
||||
},
|
||||
"ReadySlowBurn": {
|
||||
"detect_seconds": 85.981,
|
||||
"resolve_seconds": 303.95
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"firing_intervals_unix_seconds": {
|
||||
"ApiFastBurn": [
|
||||
[
|
||||
1795186070.950197,
|
||||
1795186225.223615
|
||||
]
|
||||
],
|
||||
"ApiHighErrorRate": [
|
||||
[
|
||||
1795185908.024147,
|
||||
1795185996.921485
|
||||
],
|
||||
[
|
||||
1795186042.440471,
|
||||
1795186313.1144419
|
||||
]
|
||||
],
|
||||
"ApiSlowBurn": [
|
||||
[
|
||||
1795186082.232623,
|
||||
1795186313.1144419
|
||||
]
|
||||
],
|
||||
"QueueLagGrowing": [
|
||||
[
|
||||
1795186463.2738628,
|
||||
1795186750.3954089
|
||||
]
|
||||
],
|
||||
"ReadyFastBurn": [
|
||||
[
|
||||
1795186492.702774,
|
||||
1795186804.783844
|
||||
]
|
||||
],
|
||||
"ReadySlowBurn": [
|
||||
[
|
||||
1795186501.9744341,
|
||||
1795186873.1713321
|
||||
]
|
||||
]
|
||||
},
|
||||
"short_spikes": [
|
||||
{
|
||||
"signal": "api",
|
||||
"action_started": 1795185864.820739,
|
||||
"paging_events": 0
|
||||
},
|
||||
{
|
||||
"signal": "ready",
|
||||
"action_started": 1795186346.6159208,
|
||||
"paging_events": 0
|
||||
}
|
||||
],
|
||||
"collector_rss_mib": {
|
||||
"min": 238.91796875,
|
||||
"max": 239.03515625
|
||||
},
|
||||
"collector_max_queued_packets_sum": 64.0,
|
||||
"collector_queue_capacity_sum": 64.0,
|
||||
"backend_outage": {
|
||||
"started": 1795186883.262524,
|
||||
"finished": 1795186974.814038,
|
||||
"before": {
|
||||
"at": 1795186883.262471,
|
||||
"phase": "backend_before",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 0.582676,
|
||||
"worker_busy": 1.0,
|
||||
"worker_attempts": 92.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.14903846153846165,
|
||||
"slo_good": 646.0,
|
||||
"slo_bad": 225.0,
|
||||
"collector_received_spans": 163009.0,
|
||||
"collector_sent_spans": 160474.0,
|
||||
"collector_received_logs": 15246.0,
|
||||
"collector_sent_logs": 14928.0,
|
||||
"collector_failed_spans": 402.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 250630144.0,
|
||||
"app_memory": 452972544.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
"after": {
|
||||
"at": 1795187083.355521,
|
||||
"phase": "backend_after",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 1.890634,
|
||||
"worker_busy": 1.0,
|
||||
"worker_attempts": 132.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.20124999999999976,
|
||||
"slo_good": 684.0,
|
||||
"slo_bad": 226.0,
|
||||
"collector_received_spans": 170729.0,
|
||||
"collector_sent_spans": 165537.0,
|
||||
"collector_received_logs": 16224.0,
|
||||
"collector_sent_logs": 15564.0,
|
||||
"collector_failed_spans": 826.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 250642432.0,
|
||||
"app_memory": 452603904.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
"ready_count_before_backend_recovery": 890,
|
||||
"successful_writes_during_outage": 360,
|
||||
"restored_trace_id": "60fba0bcea6e4b9fa2aead15e786b16c",
|
||||
"queue_bounded": true,
|
||||
"collector_rss_below_limit": true,
|
||||
"send_failure_counter_counts_attempts_not_unique_loss": true
|
||||
},
|
||||
"preservation": {
|
||||
"acknowledged_messages": 6370,
|
||||
"accessible_messages": 6370,
|
||||
"accepted_uploads": 359,
|
||||
"all_accepted_uploads_ready": true,
|
||||
"sampled_originals_match": true
|
||||
},
|
||||
"final_30m_sli": {
|
||||
"api_good_ratio": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795187374.124,
|
||||
"0.8992513640400964"
|
||||
]
|
||||
}
|
||||
],
|
||||
"ready_good_ratio": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795187374.131,
|
||||
"0.7983193277310924"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,553 @@
|
||||
timestamp,phase,status,seconds,trace_id,message_id
|
||||
1795180069.9570692,warmup,201,0.16097378730773926,e776267093834c7b89d8d53fd3c6f8b5,2d1f4e3e-8b85-4177-a48b-657ac153f43c
|
||||
1795180070.212939,warmup,201,0.043734073638916016,eb474a2a92eb43989227a6f21537ac21,8afc44f3-b628-47ca-a48d-c38434e6d664
|
||||
1795180070.459362,warmup,201,0.2732198238372803,0ff25933db8c40428275ec65a3905bf8,b3d8b459-e5e8-4e85-be31-8ca4aa62b4f1
|
||||
1795180070.712351,warmup,201,0.035295963287353516,cc60cc8a82014effa9f831f3ad2909d3,bf276116-3268-4256-a3ce-4f4a8b48608b
|
||||
1795180070.957216,warmup,201,0.06459498405456543,6acdfc552b2242458bb20b805a12bfdc,e45fb1d6-eddb-4636-b765-d3f1b54ef47b
|
||||
1795180071.211649,warmup,201,0.03757905960083008,96156e96fb4a462b919edffb002c1277,ebe8451e-7aa1-456c-87d4-d770b07e4baa
|
||||
1795180071.460105,warmup,201,0.009978055953979492,be2e4d671bc646e3b0ee2b882d6dd4a5,0255c592-503c-4046-9624-35e44a618fc2
|
||||
1795180071.717076,warmup,201,0.011640071868896484,c877e6f6a0dc4b0395715916ba76a283,fb9c0ad6-ae59-4cb5-9a50-df03f5fa5cc9
|
||||
1795180071.9642081,warmup,201,0.020227909088134766,0fdaa448d3964c5e9a44469c6cb68277,4a96e606-549c-4427-baab-191bb0d36e5c
|
||||
1795180072.21425,warmup,201,0.038519859313964844,43b80ef7af8044e9b2b1bc1234c15f13,8cd6c3e7-6451-473f-9259-1aca3d8ac834
|
||||
1795180072.460448,warmup,201,0.04502081871032715,dbf31c750923429095b0196c95454728,56503f1b-d7c3-4849-8654-d53bb7af63f2
|
||||
1795180072.7154849,warmup,201,0.031109094619750977,46f4da900e4a4e009656802df64f2c8f,91d88223-fdec-4c91-bca0-4d5b28d76604
|
||||
1795180072.959334,warmup,201,0.02359318733215332,99017fc0c3804148954a692be6c93b41,5579e52a-c91a-46b1-ae3b-b0475bc94130
|
||||
1795180073.208265,warmup,201,0.027540922164916992,316c362b4f4e4ccfbcd7ab3f211f8cfb,7139f79d-c498-4056-afc8-e5c1eaaaf9be
|
||||
1795180073.45748,warmup,201,0.07488799095153809,feffea2172e7461cb4171f44ef34b361,a1f19e0c-0ff2-48c3-882f-dee7064313f5
|
||||
1795180073.713435,warmup,201,0.015566110610961914,2547a0ea547441dbbe974d44b458241c,9324684d-fe9e-43a1-a84b-bae5523663b9
|
||||
1795180073.9612842,warmup,201,0.07160663604736328,00228b48dcbd4a1c84e81aa4c977d016,0f6ea533-bd56-4ed5-8c78-dab41ee30320
|
||||
1795180074.216365,warmup,201,0.032988786697387695,e807fef317064c769b2074bdb344bef4,7b5834e4-65e0-404a-b07b-a58346afb77d
|
||||
1795180074.467131,warmup,201,0.02714705467224121,c8eab445c70342b78ab3399b605b83c7,c530b3a8-c5e5-4c69-a3d9-0f0a8e20eda7
|
||||
1795180074.713984,warmup,201,0.026133060455322266,7e6544a5661349c29edf9af88d3c9fc7,3dd3ed27-4056-48ba-9e8e-21515de5d2a0
|
||||
1795180074.967433,warmup,201,0.023201942443847656,9a9720dfbe2442cea02dccf4eea97599,dcf1e137-2670-4a59-a022-58bf88cd9356
|
||||
1795180075.213536,warmup,201,0.027930021286010742,59642b7fe13d4635833fafa6207876f7,5ffb0a8f-0811-4fce-96ba-fa8e5414153c
|
||||
1795180075.459161,warmup,201,0.01526188850402832,bdbf4b92112e4787890cf366c3f67e71,a03d1041-2060-4a43-8fcf-77da2645cbfa
|
||||
1795180075.708655,warmup,201,0.12127065658569336,9e6b038f957743939b2ad4bdb3e7209f,3e944606-3b6f-4131-92ba-7b70a748f85a
|
||||
1795180075.965733,warmup,201,0.05122995376586914,9234b276a32f47a29b9fce67868601df,2b4bf449-2982-4d13-85f5-0a549a65f716
|
||||
1795180076.21755,warmup,201,0.033921003341674805,c0d749b56e5049888a86e50212011025,90afb79d-6102-46b8-98d2-b7ff3914345a
|
||||
1795180076.460042,warmup,201,0.017657041549682617,2984009087fd4f39bb9bb7cf1fca8a46,d833b693-bb00-4cc0-8aba-c116773024e5
|
||||
1795180076.708305,warmup,201,0.009041070938110352,138f1cfd9f8845098388b0ee14118da1,0c474577-7e64-46bd-b965-80e9d7cfe10b
|
||||
1795180076.963747,warmup,201,0.012278079986572266,03b2408cb3e344a49ab3f04c9607f160,6435d985-2856-48de-b33b-e335146db38a
|
||||
1795180077.208763,warmup,201,0.025561094284057617,f545f350e2064ab697c70fdf8c38f2f7,7d2ece38-e7a1-4afa-a4df-74f9e0804e58
|
||||
1795180077.46699,warmup,201,0.018079042434692383,3f245fa447364122ba33abe0f2c09179,7047997c-e173-4736-aa5c-36394ebe762f
|
||||
1795180077.71347,warmup,201,0.013484001159667969,2d3da404a33444689cda5369dff039c1,c5c4835c-4401-4f50-9554-664126393837
|
||||
1795180077.9609568,warmup,201,0.011887311935424805,9d82a52c79664331aa13a40571badfcc,1faabd95-bbd4-4d1a-8d11-410ba137b9dc
|
||||
1795180078.209839,warmup,201,0.012413978576660156,57ab3eaf40da43b780d29f19843cbfb2,65896890-75bb-4934-8bdf-99ca1f783b40
|
||||
1795180078.466974,warmup,201,0.012367010116577148,cea396ac874b472b937087e684fad26c,d762933f-ef30-4869-9d23-f015363c6f3f
|
||||
1795180078.7080152,warmup,201,0.01634359359741211,8574bb35d3d24cd1aec875b112e6cbf0,c6946cbf-85e0-4d06-927e-76f43486e875
|
||||
1795180078.967406,warmup,201,0.017637014389038086,bfa65e35fcd04d238601a48a17b433e5,5cd65d42-15dc-48d8-b965-c0b70970e96c
|
||||
1795180079.213855,warmup,201,0.02095198631286621,93c131a83a3049a8823296d254f8e444,772c2a2a-9d59-45fb-b235-3e487708fe9d
|
||||
1795180079.466801,warmup,201,0.009394168853759766,3d7c6fb3c23e4a5c905c949efb549c05,fb4240c2-55be-41b0-a2f0-f0802bdc69ac
|
||||
1795180079.713575,warmup,201,0.009850025177001953,6fc2e838823b4f56a5562957a968979f,2851c65a-763d-419a-ae13-74da0e76fc81
|
||||
1795180079.9616141,warmup,201,0.09672403335571289,5677b6efb3034fd594a53a94fdb6e4cd,77c8df03-15b3-43fc-9a1d-386f9ff3e1c7
|
||||
1795180080.210528,warmup,201,0.25554704666137695,c64242d9feee4b45af94b4a54297b725,b14682f0-d03a-4e5d-8e19-18f5544ef9b5
|
||||
1795180080.459133,warmup,201,0.04615902900695801,5722b7e2ddc84215b9dfc1dfc63fefcd,b241e81e-ac05-42d6-b38f-a1f034400491
|
||||
1795180080.710989,warmup,201,0.036536216735839844,86b110dcacc7437f8f63805c9a881dc8,c6a1ad13-e162-4bcb-a779-8a819232e4d3
|
||||
1795180080.95889,warmup,201,0.02017998695373535,ce58cb069c9b4455979a226de5bb77d4,14b20808-646a-48ee-a811-38d17f73eeba
|
||||
1795180081.216896,warmup,201,0.012832880020141602,893f8fd075df4a24adbaeb3424dd842b,b5b81a9e-9512-4427-8bd0-3f0fa5c7f6b6
|
||||
1795180081.4668531,warmup,201,0.05256390571594238,2b3598c4da6041ab812aa3fad1f8878f,c065a623-dc6a-4f4d-9614-eb13f97966bf
|
||||
1795180081.7096372,warmup,201,0.014587879180908203,8546f1a24bd74842b29ee4600bad0e2e,2b68798a-8710-4d8b-a511-995c37de2613
|
||||
1795180081.9634979,warmup,201,0.017155170440673828,925a99f02c5044c49e5c914e565b07f8,44b041e0-814c-4f00-8640-3c398c8b601c
|
||||
1795180082.2102349,warmup,201,0.023682117462158203,165c55115b8649c380b7552bae3f579d,9b01fd09-ed33-461a-8595-911e17590996
|
||||
1795180082.466957,warmup,201,0.023247718811035156,a6ca3d77dab14658b899fef8fc4f984c,7e017c78-4c50-44d6-a222-a47a48a43fa2
|
||||
1795180082.712265,warmup,201,0.015156984329223633,913d65477bbf43e4810325d8f955aa95,42cc1f17-29f7-4e4c-b5b3-bba9a9fc2c49
|
||||
1795180082.966871,warmup,201,0.014148950576782227,b2278b9f6d014d9daa209d4de38cf1b6,af0c15dd-80d2-499b-a0a3-1983237d6379
|
||||
1795180083.211093,warmup,201,0.022732019424438477,c649ef237f904d6cabbad2aa6672cdc7,deda125f-e316-4889-b197-3f099963cc39
|
||||
1795180083.465457,warmup,201,0.014335155487060547,7236502f12884b019f03115817e3cfc5,9ac3c065-5c39-4844-937a-3dbe092990b0
|
||||
1795180083.716855,warmup,201,0.014973878860473633,d5573690968947ddaa68bbd9dfc54fd2,1d6465fb-1b33-4057-a73d-f3a95cfd5666
|
||||
1795180083.962472,warmup,201,0.013889074325561523,27c7826e18a6438db0e9c29cde09d003,ac4ffbb5-54de-4b46-8a8e-d1a5bbb24725
|
||||
1795180084.216266,warmup,201,0.010313034057617188,3d133c89500445569c772981d7a6d8f2,dbc8e815-7ef5-44da-941c-a6cccdb42a1a
|
||||
1795180084.466589,warmup,201,0.027796030044555664,ed047dff153e4088b1f929221a33eaf8,f74d0068-af09-4a33-8d8c-8008480b2bec
|
||||
1795180084.716776,warmup,201,0.020221233367919922,8a65f7dc12ab49bbb8605bd6f602be4e,17a6f33a-9636-46e9-abe5-1221c741b91f
|
||||
1795180084.959736,warmup,201,0.02201390266418457,609086211d9044ee91129ca6cfaa07fd,a534b4ee-e80f-441f-b2ef-768613aa654f
|
||||
1795180085.208767,warmup,201,0.022257089614868164,ea4a411fb9c04271a5ad8e6b1f1168f9,e38ebaf2-6941-46a1-97d6-63d5202d4f11
|
||||
1795180085.4657888,warmup,201,0.03608131408691406,dfee0690d5d749988c9bd39aa51a9ec3,1a6978d7-ba7d-4d7b-9b4c-6595bd07ab1d
|
||||
1795180085.710897,warmup,201,0.010241031646728516,239f8a508e744cfdada3dd0492f45d77,1bce9275-d0a3-424c-aa80-f8d25483e609
|
||||
1795180085.964003,warmup,201,0.015243053436279297,950f416c575744038378061499ba2b72,74641199-21e6-4de9-941a-37e489252e2c
|
||||
1795180086.2077,warmup,201,0.021860122680664062,13da9df1a5814f4ca8da0cb28caeded9,c4ef5518-de9f-43b6-b05c-73a1f16f8cc9
|
||||
1795180086.464135,warmup,201,0.04335808753967285,ffc1aa67328a4b648efee25dc0f80115,e3323242-ad8f-41ae-9942-bd64702c4fa2
|
||||
1795180086.713995,warmup,201,0.013170957565307617,ff6993364f5545448fad2c42fb86f2e4,7b43da44-ea19-4ff7-972f-b48bd2b3ddea
|
||||
1795180086.958281,warmup,201,0.01422882080078125,9def7c264a2749a3a0bad35a6cec362d,5d3391d7-c86b-4789-95b1-59414122b2b6
|
||||
1795180087.211974,warmup,201,0.057511091232299805,abad7cc034f04605ae5e81a60de8014c,2bd49519-3a50-4495-bfdc-bf5dc787fd0b
|
||||
1795180087.466767,warmup,201,0.010188817977905273,0c839d1587424440b7218b6581d0e536,ecc9ecbc-c3d1-4d07-9bdf-a4abb256aea5
|
||||
1795180087.716816,warmup,201,0.03182816505432129,7c535a26c45341e4a586bd6ba39254e7,cfb59993-e9c4-44c2-b182-9b05ad6f15b1
|
||||
1795180087.964291,warmup,201,0.014871835708618164,09482390f77944e6ab2c518c0e01bca5,b65376ba-6344-4203-98f4-4b828ede7d57
|
||||
1795180088.2152479,warmup,201,0.014531135559082031,21374ca696c34dad8bd1743aa2acb6d0,2adecbe0-a793-4ac6-9ed7-d5d76b8665cb
|
||||
1795180088.4627619,warmup,201,0.014763116836547852,3bc9b4a40c1b48d4b9ad9c7f78d1fe24,b87f059a-7ddd-4471-957a-ec8ffc1d037d
|
||||
1795180088.709258,warmup,201,0.02199387550354004,b71531aab7b846b784ce55529938682e,825dca60-b7eb-41c1-b966-3708fd3c0519
|
||||
1795180088.957341,warmup,201,0.012842893600463867,522ca628e4364233b9125e64327bc83b,a0d28109-e2d4-4911-a0b2-d44a6e8a5ce0
|
||||
1795180089.208671,warmup,201,0.009891986846923828,8020a9c2e82b4342a5864ea2da0dbe2b,740898e9-2cb0-4add-b566-85c1b91a27cd
|
||||
1795180089.466711,warmup,201,0.0107269287109375,15cfdf5c30ad4ddf91b2335a7edf488a,0769beef-2b9d-47b6-9411-8398fd812edb
|
||||
1795180089.7104251,warmup,201,0.020959854125976562,7948119f04dd486aabfa051f99212538,fd8234fc-0b33-4ab8-93e0-2d81297293f4
|
||||
1795180089.963606,warmup,201,0.012027740478515625,742a4471437c49d18389b06e805a4ab3,b94f2545-4ab8-4adc-a3f0-3be6d2651cb0
|
||||
1795180090.209379,warmup,201,0.0351109504699707,485f834821ee4901800a56167fc57a58,e092a5ac-578c-4e14-953e-e1a0f35307d0
|
||||
1795180090.466652,warmup,201,0.011600017547607422,530e48d202294f02a057575c96939e09,a4d65f08-4d6e-433e-a26f-829b382a7af7
|
||||
1795180090.713542,warmup,201,0.02402806282043457,eb8528092fe340249c57b787262eaa7b,cf3c1f85-16f0-46b5-8967-ffb1ed355a34
|
||||
1795180090.958018,warmup,201,0.031564950942993164,066321c9365c4ac389b0ccc0246b74ac,ada76196-4fab-4482-83fa-b1f3b5eecb9c
|
||||
1795180091.20841,warmup,201,0.047250986099243164,6e9c7ee6a8224292a5309c9efcfc6fdd,c010493b-68c9-4411-a131-28f51549c637
|
||||
1795180091.4665458,warmup,201,0.01369023323059082,4f35ff3427584bc599fffb38720c8844,a5a00939-38ad-4653-8c3b-eb2f6b6dbc4c
|
||||
1795180091.712262,warmup,201,0.04745006561279297,9a10a79bd165470f9d80c1b1af2033e6,c24bfdf3-5d25-468e-9624-70f7dbcde2e6
|
||||
1795180091.956745,warmup,201,0.014162063598632812,c720e30747a0488fa74b22944a2cf758,15343548-7567-4d6d-973d-5915cd560bd6
|
||||
1795180092.2134972,warmup,201,0.041545867919921875,58a18bbe2e4c4fa3bbf977564dd3b070,c744740b-c846-461f-b148-dd573f0400df
|
||||
1795180092.4640179,warmup,201,0.1579892635345459,5ceb3ba17b6f4b6fa82e83dd2b45a550,55f782d9-ba6e-4fb7-a0b1-2ef0b321a085
|
||||
1795180092.7122822,warmup,201,0.018977642059326172,8cfe624196ad4a108d25effffb13bb76,37c97b83-8484-4fcb-9c0e-6cfab9469183
|
||||
1795180092.957189,warmup,201,0.07108902931213379,17f94c7584e84f8998ead6d44f40e85b,9edc0874-26af-41e4-85a6-83f49cb22fc7
|
||||
1795180093.2155,warmup,201,0.015893936157226562,4f50a2f75a004d2890d0444434b2c654,02165777-9f1c-4285-987f-eb71c5a1c4da
|
||||
1795180093.466285,warmup,201,0.010670900344848633,1e70aae6212e4411bc77affcf76c9122,7ce7a83c-43e0-479e-920e-6c45ddd097c7
|
||||
1795180093.716158,warmup,201,0.01917099952697754,e7f7d36ef7e34852a270e54064609d93,4edee8eb-99b6-48c0-96a5-f24d939999df
|
||||
1795180093.9611259,warmup,201,0.033547163009643555,aa606a9422d84863bf1e7cce2e1e99aa,ba8d2d08-9473-490f-9fa9-569c37363808
|
||||
1795180094.21668,warmup,201,0.017546892166137695,88519bbc68b745baafe3f587e050ed12,34cc6507-b740-47e1-a9f8-12fb125c7168
|
||||
1795180094.463606,warmup,201,0.02402782440185547,f2cb2077364c42eb99f97cc6e9cc411d,c84a275e-a365-404a-88b7-7619802c0867
|
||||
1795180094.7084239,warmup,201,0.06055402755737305,ae3a8b7725d94141a71f3a4bb7b6d3a6,b51d198f-c258-4c31-9fb8-699f4dd6a649
|
||||
1795180094.963558,warmup,201,0.011219024658203125,cb338827715d447281ca1cfb7baa6192,b9eeefe0-18b8-4d41-9a56-7e1a264599ca
|
||||
1795180095.2135942,warmup,201,0.017534971237182617,048f292244af4d128526d24968d99cce,abe30323-cc5e-40f3-b0c5-d2fd04779543
|
||||
1795180095.464899,warmup,201,0.02784109115600586,ca74025698904eb7a7c1b881dce81fd9,cc146ef6-43f7-46f5-b6b7-ad8f33422a17
|
||||
1795180095.712756,warmup,201,0.06147313117980957,3b9a5cc7e14846f190112798ee495f8d,582f8ca7-ed3e-4635-bd14-3f086ae088e1
|
||||
1795180095.961024,warmup,201,0.02721571922302246,3f9e74f4846747249dadb045a84612f1,b9164fdd-cfdb-40e8-9972-a7bb15331b96
|
||||
1795180096.215352,warmup,201,0.011113882064819336,230ef86450f5409f90b0f3b72f13477b,426eb3eb-84ac-4871-ac6d-edfb3edfd9e7
|
||||
1795180096.465764,warmup,201,0.010383844375610352,92977e8d8ca94f42b9f82b6760f37357,379c5abf-56ee-43d5-b7c6-9e32af7b3f83
|
||||
1795180096.713002,warmup,201,0.010370016098022461,41ead6618f494679b7b1dea567965717,481f589f-948c-4447-99cc-fde77608cb8d
|
||||
1795180096.9571269,warmup,201,0.013314008712768555,48a1a94c03674a668fe50472b6ca524c,0e64fda1-874a-49e1-9c0f-06a137d3a3fc
|
||||
1795180097.207125,warmup,201,0.03571009635925293,0855de458df949c0960a19a3d6f81481,3c40f46b-c28a-4b51-9f7a-0e3747104429
|
||||
1795180097.4657059,warmup,201,0.025941133499145508,fc09a297445c4255b337f8581fb73d0f,3b06592d-e56d-4006-96dc-13fcce9c40b7
|
||||
1795180097.711473,warmup,201,0.019830942153930664,cf215ce3ae6841678c323fafc036c13f,9f445733-c50b-4690-a50c-900e3804ef95
|
||||
1795180097.956442,warmup,201,0.011531829833984375,abba07cb6ab64b16a586f2193a8646c1,b931ebd0-fb63-4562-9417-7ac764ca1dea
|
||||
1795180098.2144608,warmup,201,0.014567136764526367,e8d2e2539d774d88aaa2c4473977c67e,df144bb7-3ed3-4e30-b65a-194064b53183
|
||||
1795180098.4572449,warmup,201,0.08848404884338379,6bd7fadb57c9417fb94e1a3419ab8cc4,271ce552-30fc-4efd-8cca-4eba7419763f
|
||||
1795180098.708143,warmup,201,0.0271909236907959,5afd4658539842928dacb103fcf9de04,cce0b416-63c7-4dd9-9493-f8fe4a860a10
|
||||
1795180098.96469,warmup,201,0.08780908584594727,b62003bace334091a3b8aa60404be850,bba8ff6a-4a76-40e6-bd6a-281ec6a96868
|
||||
1795180099.2078679,warmup,201,0.02692413330078125,9e650dbe0af2454eaa8998c57d0cf42b,481bd894-496b-4266-b6b2-0e44cc1426a3
|
||||
1795180099.4654238,warmup,201,0.012391328811645508,8b69c1152c28444b8d903d040f63d724,d66a387c-8a71-484d-878c-c6039df6c7cd
|
||||
1795180099.716196,warmup,201,0.013508796691894531,08a4e30d9cd040289e88f1f32e6a6cab,1b1ecdbe-9293-41d7-842a-873e3ae2c49d
|
||||
1795180099.965012,warmup,201,0.03931689262390137,f81e66c40e4c40c0a3574da6f042285f,9c2c656f-1486-41ce-b120-0a55526c0d95
|
||||
1795180100.2069771,warmup,201,0.027806758880615234,29c3c90fad8a4ed781afec70fc431013,018459b3-7916-44b2-82c0-c7390a0b7ac1
|
||||
1795180100.460327,warmup,201,0.019770145416259766,a2d50f5e1f124fab88c3ff5019c37652,89eea49a-a704-4a3a-af3d-21309c592214
|
||||
1795180100.713459,warmup,201,0.022834062576293945,dc517a51ea684bc993695fce6571b5aa,87ace08c-2856-4ef8-8482-1c81e698894d
|
||||
1795180100.9633029,warmup,201,0.011710166931152344,ef190508073f479a9d57bdb4c9b9c1e9,64bec410-474f-4412-b34f-8aa8d7b9e3b4
|
||||
1795180101.216318,warmup,201,0.01240229606628418,7ee886cbc5094be3881d8882c3dfe9a5,f5fa9bfe-6f92-4b28-a439-7937370a2ed5
|
||||
1795180101.460207,warmup,201,0.011929035186767578,e1a6eb07742d48729e6d06a3cef34df7,3530d4ee-e604-4b01-8f12-7b1441f2d50b
|
||||
1795180101.716116,warmup,201,0.010046958923339844,0c100de439214d3ca293b8917cf9c575,f725ce7f-93f2-4e97-be37-d41ef73982a9
|
||||
1795180101.956589,warmup,201,0.01887202262878418,bd9178c605734da7bf53e692c7fa269b,a085ee86-d766-4b74-8568-5d3e1d9ceec3
|
||||
1795180102.2150679,warmup,201,0.04256629943847656,7ab9325f6b5c4364ab96c9369224cf1c,6835f67a-7cd0-4d11-ab98-e40d44693689
|
||||
1795180102.462175,warmup,201,0.01769113540649414,8f392006bf984a59aedef30ee800156e,26aa8e16-eba7-4696-b32b-8dfb3cf13d74
|
||||
1795180102.709732,warmup,201,0.04403996467590332,344073f83eb3454faf345c87ae4c89f6,333d76eb-d994-45bb-a987-c2c146bb8562
|
||||
1795180102.9586382,warmup,201,0.015874624252319336,ce2c60ba55fa4829a3a477511b333ffa,fd203964-ff78-425b-8c1e-881e0323ec05
|
||||
1795180103.206547,warmup,201,0.013957977294921875,a577df3378ff4b5fae4c2e0f7bb8bd24,929b0cb8-2692-4995-8dcb-5c61cad260c3
|
||||
1795180103.465699,warmup,201,0.06361103057861328,b1716a8aaa4d474eab1ea16c7ef710c9,f076ab04-d83b-49be-b026-73bf0c205d02
|
||||
1795180103.7078888,warmup,201,0.011960029602050781,5bf8e68832564e3d8b0815af5d9f4326,6928bfbd-e29a-441f-a048-635a466c73f1
|
||||
1795180103.95976,warmup,201,0.014333009719848633,d8bdd91a857745d6abe1ed3e19aea098,5e559263-dd52-4050-a13a-ae1441a79556
|
||||
1795180104.210141,warmup,201,0.0119171142578125,ec5ed7bd28ba4cec925e85e563b22928,3170e5ec-95ff-4f0d-b8af-9b8803663b1b
|
||||
1795180104.462445,warmup,201,0.013860940933227539,1cceb6665907453fa5bb35ae531e06ae,099f91a1-f991-4315-9816-1b8dac80fd90
|
||||
1795180104.716085,warmup,201,0.027819156646728516,b569df314b2d4c28bdca00cc5638ec82,457f96f9-fe3d-4086-b0c8-bc1148502545
|
||||
1795180104.9571462,warmup,201,0.010566949844360352,ee53243220954c65be8a449284fd22e8,148c93de-1a65-48b6-84f0-ae15b9be6d50
|
||||
1795180105.2108119,warmup,201,0.01622605323791504,a8009e01d96f48dbac64193fd0fe557c,200a85ae-6619-4391-8f21-56ce2877b3da
|
||||
1795180105.465821,warmup,201,0.01184701919555664,aa99ea688eab46af83e302c28ce7d756,2ab663cf-6898-4cc4-9e43-8e40ddf480f8
|
||||
1795180105.7093928,warmup,201,0.051905155181884766,0aaee39c361546b3849fd152947d9cae,8272cf06-6e91-46a8-a1d8-9d08e65c331b
|
||||
1795180105.9574258,warmup,201,0.016104936599731445,e862f37799a8410686508c3ed816d7ba,9de19f12-24b5-448d-949f-ad8adbc32650
|
||||
1795180106.215368,warmup,201,0.007997751235961914,558cd74d71da4cfeb1e6785e4678b33e,b8b88a48-1e5f-4f27-b46b-edf215c0e51f
|
||||
1795180106.465971,warmup,201,0.016330957412719727,8ca7061c259749d890c2ae7a24ee10ea,a0e5fd6c-1db4-4898-987c-16f37224951d
|
||||
1795180106.7134972,warmup,201,0.01929187774658203,96db26073ac04a9c8483d94d36d648c5,ba3b1aaf-9966-443c-8bad-8f3f5cbe772a
|
||||
1795180106.965224,warmup,201,0.016424894332885742,49c438ff3cc34374a64df5a646ca0064,c04e7e61-6934-4b15-9f27-6992af1e740c
|
||||
1795180107.2087429,warmup,201,0.13824105262756348,91a3a6e513ef48e091e5fec6cadc2651,c15ae37c-a0ca-4002-94ab-63c613b8a3f5
|
||||
1795180107.465683,warmup,201,0.06748700141906738,a0b0fadee95b42d9846c6bc19f6a3425,fe21da70-979c-4f25-bd7b-ecec462a4f6c
|
||||
1795180107.716597,warmup,201,0.043663978576660156,d0a18a5e9d3844609e1846de52fdc334,09b03827-9758-4c19-96e9-02245b95408a
|
||||
1795180107.9577112,warmup,201,0.05071878433227539,8bc17d2ac861457d906edf6bb733aaa6,d40b8a9d-314e-4e6f-a235-53179160e38c
|
||||
1795180108.215978,warmup,201,0.016988039016723633,b9d90c9fef9b4cd2b40f0d01f960a800,c2cad002-8df9-440b-8d06-8e9edb144d2e
|
||||
1795180108.456348,warmup,201,0.013724088668823242,dd7f64e94a8e49808c688bc6e7705f9c,49bf1186-56cd-4c5f-962f-c1cce4115bf9
|
||||
1795180108.716038,warmup,201,0.014075040817260742,46027f62899f44aeb5b997155d9dacd8,7800d167-6a9d-4299-b585-a8e422b6ed15
|
||||
1795180108.9656162,warmup,201,0.029653549194335938,c0328e86f791432aacbd1351a7778276,c10794fb-2fbe-4254-b6d1-68c662d03d3a
|
||||
1795180109.2134242,warmup,201,0.011368989944458008,78d77602d48648b9bb473b838ba80d94,0ffa1fa7-ca44-40ef-906b-91d7e6adac7b
|
||||
1795180109.466222,warmup,201,0.011855840682983398,5a4539f748ee44139ed31d0523512e4a,03ed8cf0-6572-4420-a46f-f31c353d5ca7
|
||||
1795180109.711676,warmup,201,0.016754150390625,ada1f3a756354017b1b3606e73a5e568,530d5f4e-8a90-400c-bd80-49c3282c6409
|
||||
1795180109.957644,warmup,201,0.011628150939941406,069abe6e4e40491499fb9c0e060b1f02,fd454afa-45a6-48a1-acbc-a30f86ea1611
|
||||
1795180110.209734,warmup,201,0.01928114891052246,feac37a20259432c87e036d5a3736f05,6c0e6316-6d64-45ca-82c8-a45d07998b22
|
||||
1795180110.4659588,warmup,201,0.022558212280273438,83931cf8c4304d13bbbcb09fcbd09a2c,fe5bbefc-b090-4206-940e-53fa504766b7
|
||||
1795180110.7111838,warmup,201,0.026392221450805664,c86accc4673a4021b5846f199e507dc0,aaad07ac-8808-4d6a-96df-28b4fe06feba
|
||||
1795180110.9606888,warmup,201,0.02469611167907715,f2617744f0374aca828bd5e16a522317,a020e190-4f8b-4ffe-9ba6-454b67dab903
|
||||
1795180111.206548,warmup,201,0.025948047637939453,d08ad39890994932b23b2e165e0cab6a,26fa8160-8bf3-4577-be99-c151ae1f91f4
|
||||
1795180111.464929,warmup,201,0.021355867385864258,4ff40371a3fb4209a7995e9993296026,6f20343f-07e3-4fb3-8413-55ca6218b45f
|
||||
1795180111.706388,warmup,201,0.011980056762695312,1ba5d021fa704d43a494e01106455fff,889d6919-d410-4fb9-8e6b-d0313916c9c4
|
||||
1795180111.965568,warmup,201,0.021121978759765625,9026ed4c64eb4cfe9d358a97604851e7,4022af74-df48-47e2-a3a5-ce4c4eb29cd8
|
||||
1795180112.2073479,warmup,201,0.016837120056152344,c02d63b9da53468cab2945d158c88217,43c2697b-fa58-47cd-8aa8-0017553f1a0f
|
||||
1795180112.463627,warmup,201,0.009462833404541016,0b96e9c8026b4acfa6803e63f5d78270,61b8558d-3fbb-4fb2-a19e-143e173783b8
|
||||
1795180112.713996,warmup,201,0.015658140182495117,bc6d171122fc4a82831317d5081c8c71,2564d01e-c563-4c1b-8d48-9f4c7625f03d
|
||||
1795180112.959977,warmup,201,0.03197908401489258,26c142b3d1f346afa03037a9fd03f264,f21f69d9-3855-492b-8e12-c1c3173ff82e
|
||||
1795180113.461932,warmup,201,0.3604569435119629,e0de43a93b9747ed8135916b9161edb8,88a841fe-a14d-442d-96ca-f9ccbbd90a85
|
||||
1795180113.963574,warmup,201,0.0680840015411377,7f5ddf5140ca41eca9f99005e3be922a,1d915906-50a8-4e50-ad96-d955eae5bba0
|
||||
1795180113.2087371,warmup,201,0.8404510021209717,266ef7a83b1f4ac6943f15629354bc1d,46a8f038-4ecf-43c8-af7b-a76681b0b7ac
|
||||
1795180113.714091,warmup,201,0.3419630527496338,e3408be2e18243fe952dffc3a29d9f76,6bca83aa-886a-4e16-87ff-c4cbe343b04b
|
||||
1795180114.21387,warmup,201,0.04325413703918457,531bfc2b57cb4a4abd2bace25b1b206a,56501d24-4338-4a51-ae9c-d571359a5a3b
|
||||
1795180114.459767,warmup,201,0.07159805297851562,704037cba11f4f87954b914ca2dea4d7,9b7e9ffa-1018-4469-95ac-ae45816e1db3
|
||||
1795180114.713313,warmup,201,0.013693809509277344,a8aea998a4a14b068536acb95950bd6f,ea6691a4-cd8b-4aba-9200-27596e23d8f2
|
||||
1795180114.9657989,warmup,201,0.019134998321533203,f16be51009bf4267afffe901c6f9ec25,15bd426d-2b24-433f-91a9-b50b78a05afd
|
||||
1795180115.210476,warmup,201,0.015871047973632812,8a59d5636bd849c195436327cf03d6bb,7a92fb63-0ea7-4b97-97e7-627217b57b48
|
||||
1795180115.4618099,warmup,201,0.017672300338745117,8d0924d1e80648d99ad4b02f3743e748,ae81b2fc-f535-4164-9d0a-849757f3de7e
|
||||
1795180115.7133892,warmup,201,0.23829078674316406,ee868fc316c44711899a2a7eac1f983f,d4d6bf84-798f-4719-8ce6-7b4f71e3edea
|
||||
1795180115.963288,warmup,201,0.06793808937072754,be960e7d72fb41d686edb4c1519cd34e,a4a8dbb8-dd44-424b-8d44-94338319fc2e
|
||||
1795180116.216624,warmup,201,0.011794090270996094,f1becd1850aa459188902bbfe85dc317,195060c9-3ab9-4947-aa5c-90fc6e708878
|
||||
1795180116.465261,warmup,201,0.010736942291259766,ccfe2471b6094bdc98d8f7c7cc5215c9,8598b485-22cf-494f-8e7e-093045e91e0a
|
||||
1795180116.7087119,warmup,201,0.019225120544433594,6490a5b10731467fb70b893217fbe7a5,fdde1515-647f-46ed-be62-61a32f012d97
|
||||
1795180116.957767,warmup,201,0.012019872665405273,c0f622e8a99846f1897ce6d9e4d24109,2efc878d-0126-4ee5-99c4-b75f1b08209d
|
||||
1795180117.2127738,warmup,201,0.023031234741210938,245477e2a0714376b674cd53912ff0cc,7396abb8-7d06-47be-bf35-caeb9104a6b1
|
||||
1795180117.465743,warmup,201,0.03672194480895996,b699aef6a81a4583ace9caf63f684ad9,5b27c6e2-7533-423b-ad6a-6ffcef894efe
|
||||
1795180117.7158148,warmup,201,0.01888728141784668,162f7b95191d40f59651888ee71f53a7,72779180-5849-4054-8d1c-e9f50e31dd53
|
||||
1795180117.96484,warmup,201,0.04443693161010742,41e9e30f1cae4d4bad5e3e51fb7fe6e8,a9de2cb7-13f6-4d4e-8418-8394733c75ef
|
||||
1795180118.2133079,warmup,201,0.023710966110229492,3698080dd9de4cb4a3b9087e6f736e52,00a5f651-45e7-4306-90e3-034890c0810b
|
||||
1795180118.460449,warmup,201,0.029680967330932617,ff85e90c255c47d0a8648258d71bef3e,cfc32a1c-8eef-42bd-bbb5-7cf95dc1971e
|
||||
1795180118.7111292,warmup,201,0.02736687660217285,cc2f0f325db64165b0e63db66ef2e912,946f661d-727d-420c-b2c1-10ccc5721ccb
|
||||
1795180118.964362,warmup,201,0.014344930648803711,cd431bb740694c60a71b0c05f8af6974,0ffe6ffb-c402-4b6a-87c3-9bb8094ff16f
|
||||
1795180119.211197,warmup,201,0.013164043426513672,d57df0e48f9848a0bcce0b113bd85726,4bf5b6f4-b8dc-4337-91e8-06d9dfb75af7
|
||||
1795180119.4637692,warmup,201,0.011598825454711914,86002fec5caa4665b801fcf8dcb2e433,f863f2b4-5df2-4195-b2c8-9b153c392a53
|
||||
1795180119.714405,warmup,201,0.024904966354370117,75aea0e8c93440a0b64f74afd2bab8e6,29929580-e108-4bf9-abd6-845383375440
|
||||
1795180119.9656231,warmup,201,0.024814844131469727,f552643138364b199afa14b6418efd65,963ac84e-6238-44d6-90a7-959c7727efcf
|
||||
1795180120.209861,warmup,201,0.019379138946533203,301dbfe0ea9742d8a6e972e148164e86,7b40e23c-22a1-4645-87d7-2cd7ecdab5e0
|
||||
1795180120.460503,warmup,201,0.012269973754882812,56d19575ad6b4a159d29b2f28035a292,ce004c67-86e3-4611-b2d3-410c60eb29e1
|
||||
1795180120.707459,warmup,201,0.016863107681274414,eb32c8837103450590381b57d9eb8a29,d4ac9a5b-353b-4cda-a097-06c588a077fe
|
||||
1795180120.956764,warmup,201,0.03303384780883789,1a549a26298f4650b8ff2effb4231aa3,304dae29-1b9f-45c3-8f80-c3ee5ab8291f
|
||||
1795180121.206913,warmup,201,0.01592087745666504,7d5af7fc0b024ea5825a6ed2a2638e50,5e90cd2f-4070-4810-a1a3-e27604dcf771
|
||||
1795180121.456815,warmup,201,0.011278867721557617,76f880ca36c241d888ee2524fc8b2b3d,8d9e023a-99d1-43ca-ab5f-3dd6d9a5df32
|
||||
1795180121.71327,warmup,201,0.025320053100585938,3df813a8d9c443ffbb44b524bbb916e8,28c64eb0-cc2e-4f71-8ede-8e852d15c2c2
|
||||
1795180121.95684,warmup,201,0.00838017463684082,5531422685b943a6a369e608c058fc57,591f8ae2-343b-4273-a2f8-b6cf48646f32
|
||||
1795180122.206609,warmup,201,0.00858306884765625,bb7288aba8434bba9fc41666e0904fc1,24f71b02-74a2-4901-8200-9cd545dcd2c3
|
||||
1795180122.462899,warmup,201,0.015925884246826172,69d4f4e4b7544c409204aec3d83da9e7,e74f7431-65b2-4e35-853e-cec86896c78e
|
||||
1795180122.709456,warmup,201,0.010297060012817383,7ae458c18ec5447c8ac6c6118bc18fcc,a090d8ab-702a-4b3f-a0ea-d4e383b45cd2
|
||||
1795180122.964902,warmup,201,0.013642311096191406,a7a71c5177a6460a8c2b7a8a5127a8a5,bf2acb88-b590-419b-8578-3201c3381a73
|
||||
1795180123.2154398,warmup,201,0.012851238250732422,b41e6dd0417040dfa5ef0bd7a51d4d5b,4486c43c-b469-4899-9d82-2d6afb8f03c3
|
||||
1795180123.455821,warmup,201,0.013101816177368164,0e0788c63dd3458a988dc582fc97093e,0914e390-c13f-46b3-bcdf-998b9deac79b
|
||||
1795180123.708153,warmup,201,0.020342111587524414,8e3ad121c9184193b84eec5c7700ff37,aa11cd4a-7649-47d7-ad4a-4daba45c64a9
|
||||
1795180123.955742,warmup,201,0.05740213394165039,c1a8efda79ec44e0a3c108a65ecfe280,70604d3a-50e9-4c70-9337-04b65fdc4809
|
||||
1795180124.208821,warmup,201,0.016861915588378906,cd2340f1dfd248ebb7d5356f79f8f810,561a47d7-b772-43a0-9033-1868fa5ac4aa
|
||||
1795180124.459167,warmup,201,0.08424186706542969,ce5f4ebaa39a48cc9925177ddd46b951,60f54ad3-215d-43a1-8118-6c72a5efa4c3
|
||||
1795180124.7120728,warmup,201,0.0500941276550293,12adf6cb9cfa46e4b59d0d827ac86be7,f523c477-67a0-40e4-822b-bd6cda069068
|
||||
1795180124.955792,warmup,201,0.46666622161865234,cde8f775be3e45d09e4f031ba0836095,ac5ed17e-b55c-497d-a88a-93d5612ac03e
|
||||
1795180125.2137432,warmup,201,0.21810579299926758,7789893b847749a39412cb64dad122d0,c7c84df7-5b9a-4f26-a31e-ebe6ca26d14e
|
||||
1795180125.4653718,warmup,201,0.4004700183868408,4c692c0c992e4cb4983d702cb7f2906d,ab70beeb-6847-4079-a880-272308582b3d
|
||||
1795180125.7116349,warmup,201,0.2136549949645996,4a952313b6c241e7b24648f925c91c87,396d5aa0-9d6d-4f66-b11c-47654ba5060a
|
||||
1795180126.20828,warmup,201,0.022182941436767578,4b2f49632f6f4378a862a76bf2b0a11a,287778ad-cbcf-405c-9781-76c32785d4f8
|
||||
1795180125.957431,warmup,201,0.4076690673828125,c19fbf5dc41f44678bf65a75718f4763,ead5c9c8-736b-4cca-a826-797020bc4952
|
||||
1795180126.465112,warmup,201,0.11259102821350098,35523a266edf48bd943263384c9787ad,70c1baa4-3067-492c-97e8-b70ddfd400eb
|
||||
1795180126.70634,warmup,201,0.1282970905303955,08db262afecd475faffb5c5151eaec0b,5a306129-f313-4586-afc8-6ab0d70302af
|
||||
1795180126.956701,warmup,201,0.08037590980529785,8b66a1e2b3fe4bc5aa66698dc7534c95,58c38c8b-047c-43ee-8e2e-a105acc3960f
|
||||
1795180127.213266,warmup,201,0.0364232063293457,023627bd9b8f45bc8335c9d02ee6a0e2,c209d4bf-212f-4267-b115-8456a3b6cd8f
|
||||
1795180127.460497,warmup,201,0.018844127655029297,d66c88144827455095a11c5bb3a8ab75,74a961b6-8a03-4592-8477-8c79765f6135
|
||||
1795180127.70784,warmup,201,0.013520002365112305,a5dec999bcac4e60ae5bc69113c4b97c,0694641f-063a-47ab-ba69-446a24b0302d
|
||||
1795180127.9577322,warmup,201,0.014217853546142578,2968ad48558e4ed5a5a282176ce12a28,211849a6-68f0-4974-a009-8f3056aac6b0
|
||||
1795180128.2072031,warmup,201,0.012610673904418945,8e65ac12d21243be946656474b9be0a9,e612a563-7d2f-439f-a64e-6ad7902b758b
|
||||
1795180128.460282,warmup,201,0.023057937622070312,66f10fbc943e4496a3ff400a3e46ed0a,eedc2fc5-c6e8-4613-88a6-77a6637c4584
|
||||
1795180128.713231,warmup,201,0.03037095069885254,31fd1195cb3847d08974df206fc6b0d7,c45a7609-ad92-437f-889c-ce7962d075d8
|
||||
1795180128.956662,warmup,201,0.028537273406982422,9e68be6d44d64037bf3006185c2d5d23,4d731559-f9ce-40c0-95ef-8633fadcdf99
|
||||
1795180129.211051,warmup,201,0.0380401611328125,03b2f65668d34f15b6c9546be47fec86,94b225fe-050b-4be4-b9bc-d85c7180abd1
|
||||
1795180129.463252,warmup,201,0.03919506072998047,9d7008be8e4b4e6b84dda2204dd3a0f7,ca4fc962-d814-42f1-81ef-5ac07d99f4e7
|
||||
1795180129.705261,warmup,201,0.01246190071105957,5b7a9e05ef18435caae5c26fd0fbba55,c780199c-588a-4089-8e1a-b9e29040dc94
|
||||
1795180129.958072,warmup,201,0.03585696220397949,80f818ad28014402851cf0dc4fe6b6fd,70fe8691-9a92-40aa-aba3-d66c7eaeb224
|
||||
1795180130.2058141,warmup,201,0.09956502914428711,29d9e9488c834b89b5811683ef978cdf,72350268-eb1d-48f7-9c64-20b973f464b5
|
||||
1795180130.465339,warmup,201,0.025896072387695312,d7b5a37e9f774ba09b755dc347a6a543,2abf41cc-ed1d-42fb-bd0b-c4dd09ac95a1
|
||||
1795180130.7066429,warmup,201,0.011258125305175781,138cf5cf556a46bd8c5996af9771a756,fcf61d52-05f2-40f8-b719-028a7d3799cb
|
||||
1795180130.9631832,warmup,201,0.013113737106323242,6cd0c7e99e144c189130e48bd586ebc2,e384af86-879a-49eb-b88d-27fa49ae5387
|
||||
1795180131.2116659,warmup,201,0.03482413291931152,4bcea84186e24a79a32ea26db27b9110,652c07d3-b5a2-4b5a-9739-34d2340bbedc
|
||||
1795180131.4630752,warmup,201,0.01401972770690918,dd7c079671544c79848adf21e81287b3,04f4b354-d35e-44c6-9cdd-5c0701c50001
|
||||
1795180131.707235,warmup,201,0.01443791389465332,7f24ba99e54449eaa6dd61923148169e,2620aa56-3e44-43fe-9f07-9608321b14d6
|
||||
1795180131.9631941,warmup,201,0.010252714157104492,5bba090e4ec542649013171deb048bbf,5cb08e95-e5a4-4ab9-bd2e-57c5f7024413
|
||||
1795180132.213757,warmup,201,0.01481318473815918,e7d645f2882348cb9e340b8993c8bcdc,b82f4159-b720-44f7-bf06-8d5d91c7733a
|
||||
1795180132.463203,warmup,201,0.024817943572998047,352c5922524b47a386900aa4e676ee34,6c286b53-7960-4d83-9885-04ac3de75eb0
|
||||
1795180132.7094731,warmup,201,0.008898019790649414,f4035257b55c41d5921cfecea8966176,574fb6a6-2aeb-465c-831b-a64ea30727c8
|
||||
1795180132.956722,warmup,201,0.010103940963745117,afd60764b6934ebd8a725d78bd3f141b,7972d992-f8a3-45d5-9bb5-4314a6553f51
|
||||
1795180133.206574,warmup,201,0.015628814697265625,1721768d2aa64664ad3e327db144300a,1edafeaf-e173-41b6-8c3c-83f18a8166c1
|
||||
1795180133.456523,warmup,201,0.011080026626586914,f15339e1ec4d4a6ca8e331011f988549,f72cc398-eaf7-4d27-8b08-c301779f1dfa
|
||||
1795180133.71329,warmup,201,0.013360977172851562,c96a9745a1b74d58b3dab2cd5645c4a4,4ac57986-d879-420a-b1e1-244e561e2452
|
||||
1795180133.96325,warmup,201,0.0173952579498291,6f2a1136ff1042d58b8f6cd077189097,cc1000c0-ad46-4409-a37d-6ff7df3275f2
|
||||
1795180134.2074082,warmup,201,0.013551712036132812,1c33f13207324f7391e00ac92f57a94e,d2e5b4fb-af9e-4279-8584-8bf2c0d76a29
|
||||
1795180134.4623308,warmup,201,0.007900238037109375,34e1fdd2284a4dc388d43c3da2725df8,880018cf-c8b4-4395-b5f5-fa1c8fa65273
|
||||
1795180134.7132852,warmup,201,0.011044740676879883,6403402a1ce84f2f8806a914dd8215b5,61f00d79-ccb5-494e-bd71-d862ffeef5d1
|
||||
1795180134.9608378,warmup,201,0.02025604248046875,0894586e34c54ab697c597b7421d034f,3244ed3b-f15d-472b-9e3c-8d3e924f5ff6
|
||||
1795180135.207734,warmup,201,0.011043071746826172,1b06517ab2a444179728fb8dbfca0827,03d2f9ae-3ebe-4dee-a81c-6b6c406f9d2e
|
||||
1795180135.455019,warmup,201,0.013014078140258789,fd6c4f6cf4ef4e1db2ecda69d20735cf,89bd0345-ec51-46d8-a006-ff0a2d410b5e
|
||||
1795180135.712738,warmup,201,0.10438084602355957,8f09b5249863497e94b4ac5dd86ba4a0,c23262a9-cc13-49bd-8f23-795dccc65e49
|
||||
1795180135.9571142,warmup,201,0.018002748489379883,f075fb7c0d434b67a026427a792aeec0,351ccb0e-cc60-45bb-ab0c-8ec9d6340b0f
|
||||
1795180136.205044,warmup,201,0.011342048645019531,fc8eb864844d49689747e262be617164,f250da35-d8a5-4b94-af66-183bb0c14d81
|
||||
1795180136.463886,warmup,201,0.022532939910888672,1e2cf1399e5947a8a3d01c53e008b506,346b4b23-4597-4a11-8206-cd77d95da8cb
|
||||
1795180136.7051678,warmup,201,0.0228271484375,1e97029cac1f49d3936d82698cf0f18d,98396eee-0cfe-4362-b38f-479c9d56121a
|
||||
1795180136.95592,warmup,201,0.01760697364807129,fa15a0606f634b378faef6d7851b21ac,5c6bad4a-710f-4f13-b688-1c694c8ba23f
|
||||
1795180137.2061172,warmup,201,0.012992620468139648,ec2ffdb4bad8408f8203f6c45fbd2bfa,722c2c2d-e8df-4960-ad16-ba0705f57818
|
||||
1795180137.4616551,warmup,201,0.011270761489868164,e03220cc975a40b2866f11bec428e421,18f123db-6131-4d5c-bce5-8cb6857af849
|
||||
1795180137.7097511,warmup,201,0.01858997344970703,e38529e88ca9466aad28b852ee566d05,d6fb2504-540c-4455-8431-c5a800f7536b
|
||||
1795180137.956691,warmup,201,0.015127897262573242,7ae2111789d54a03992356796a309d08,1f023142-499c-431a-a878-710b377794c9
|
||||
1795180138.2149088,warmup,201,0.020531177520751953,5775e215d1484f9ebbb21fd79036e3b9,5f8dc182-154f-4d6e-8720-6475c7823c52
|
||||
1795180138.45882,warmup,201,0.015606880187988281,e0f4da4791f041d0a1b626e48732b09a,8349e524-f496-4d09-bc55-b25ab82ac628
|
||||
1795180138.7100549,warmup,201,0.012995004653930664,c3461393466d4972b02dfb9c01957188,5bde5b66-1212-4260-a66c-5f7657e5f10c
|
||||
1795180138.9580622,warmup,201,0.01614689826965332,57c5d3fbe0be4fc49cb0fdad8aadfa92,cdf0eecc-6a30-4541-b9f5-ddef849546fb
|
||||
1795180139.2118132,warmup,201,0.013647794723510742,767e969ca9014b6c9996531ce559df61,88609695-abdd-4bad-96db-9fe814702590
|
||||
1795180139.4632142,warmup,201,0.009091854095458984,f86e3591b7e64a71b5553f371b7f40c9,de93cccc-76d6-493a-ade7-3ea0a0d5fb71
|
||||
1795180139.714897,warmup,201,0.015331029891967773,cddf319c99ef408c82bfb4d343ef2635,8fa78445-3d8c-45a7-96ae-01a941f020f0
|
||||
1795180139.9627159,warmup,201,0.01434016227722168,3fcf4fb003844ec8bb54ea1f5f733efa,6b4ac399-f00b-4119-ab42-0af7df3f357a
|
||||
1795180140.21356,warmup,201,0.049919843673706055,78da2033cd6746b3aacecb1a52a94bff,43f75c5a-7986-4c87-b2be-643701e8a30b
|
||||
1795180140.463584,warmup,201,0.013402223587036133,e9561b1d921a4901a0d76129b05447a6,0bd74017-3e94-4f66-8019-b964128d33be
|
||||
1795180140.713254,warmup,201,0.016618013381958008,c8478479d45f43b6abf82abeb4fa83ab,1f35207e-990e-4b2b-9b12-8b2ca300e495
|
||||
1795180140.962937,warmup,201,0.019067764282226562,51f2dcc6fa61439d9b90a247c2f15144,1a4ee6e5-a8aa-4b57-ac2c-f6d7a8605e02
|
||||
1795180141.214133,warmup,201,0.011148929595947266,acbb62edba674fdbaf5c001a80efa143,1bed9e98-06d0-44aa-99ef-7a0ba428d72b
|
||||
1795180141.462243,warmup,201,0.017611026763916016,a80c5bdac3b848ac89ab8d1132f29295,81969480-34ca-46ce-8724-090271f2d3ea
|
||||
1795180141.713085,warmup,201,0.02792811393737793,4a1c8d5e101e457aa1bb3532c62715ff,2a4755eb-58d0-4e0a-8f04-c7e085bb720a
|
||||
1795180141.963087,warmup,201,0.01384878158569336,adc2b0e3598945b4b1b6831405aea2f2,4437fa0d-be96-41dc-b191-332f69d6aadf
|
||||
1795180142.204875,warmup,201,0.01084589958190918,09891c3bb41d4df9836d56c0ad0e942f,bca6e934-63af-4bb9-956f-e53c7205efd4
|
||||
1795180142.464761,warmup,201,0.010806083679199219,660ab3f0b0ff49debf10a44783abe0fc,b83355bd-9cb9-47cc-8a37-295e91755f76
|
||||
1795180142.71471,warmup,201,0.00988006591796875,3ba7601657da46a588e8a6e9ef71cd8b,20c7f9a3-b284-4af0-a23b-c16479768a75
|
||||
1795180142.959651,warmup,201,0.015625953674316406,eb71302e239744f2a759cd4f4902a058,1c3ae2d0-920a-45e2-8914-0e71924f5a05
|
||||
1795180143.213386,warmup,201,0.009329795837402344,ca97cb82e52544f6a5c4400f7bb5f3f5,bea54320-8738-46fb-9f20-0ebc92a9d599
|
||||
1795180143.463989,warmup,201,0.013462066650390625,9ac1e64e71e84d41963aceb41d3c9331,959cec4c-2063-4e65-ac56-29b3733739a5
|
||||
1795180143.707256,warmup,201,0.0188601016998291,d18b958f46c247569415bfab8c827026,5acac01c-4e9c-41c0-8153-81cfb3f4879c
|
||||
1795180143.961851,warmup,201,0.01336812973022461,2f1a398627144fbdaf3f2e478d33b28d,e3962383-7d5a-4a92-877f-249f3bfb0f41
|
||||
1795180144.2063081,warmup,201,0.011801719665527344,39a64faf76494e99a03a888cc7e2e4c5,042bd8b2-5e39-448f-8fb4-0f5278a920c2
|
||||
1795180144.463094,warmup,201,0.012636899948120117,db742f1e32d6433fbdeb0ddd3da29f97,f5fd8b85-e55c-4e8a-8870-1d645bb46a20
|
||||
1795180144.7092211,warmup,201,0.023552656173706055,64f7b49a036248d1a70980d117a0264c,019eb5a9-db89-4439-b044-849894838ff4
|
||||
1795180144.960939,warmup,201,0.01747727394104004,a2f1fee7c68847e5ba5d2cb6e950c8c9,6992a0cd-0f41-4045-99be-c229b8450d68
|
||||
1795180145.206008,warmup,201,0.02232813835144043,e5be693bf9654a7993d12aa16398e524,8061e634-e325-4d3e-9c3f-79f13edb06b8
|
||||
1795180145.464622,warmup,201,0.025964021682739258,89689b110ecd442f8d823fd5ad1ef546,d225696c-a1ab-4f82-b65b-0ab6f411b87c
|
||||
1795180145.714587,warmup,201,0.013881921768188477,e5b573ab906d4d47bc283bad0c283bde,d0cd8d44-e388-44dc-9939-31448705398b
|
||||
1795180145.956403,warmup,201,0.017344951629638672,1b12f10f44a54f87a2f82e80da2ca146,1d61c6a1-87c4-4622-9f7d-a1c5b97f3f1e
|
||||
1795180146.205685,warmup,201,0.01906418800354004,be5614f0646e4c6c927704b32c216845,9ba1718c-e5de-4c71-a971-e1a06705f187
|
||||
1795180146.46311,warmup,201,0.023369789123535156,070ea00c630b49c39cd773bd022a2440,abcec0a3-3c91-4598-838e-9e4b098828e4
|
||||
1795180146.7096572,warmup,201,0.11615276336669922,134e4f4d39724ce2b453908df6ee3c32,6c9588b6-3e39-4335-ab45-76980ca22058
|
||||
1795180146.962498,warmup,201,0.015411138534545898,bc00ee055e934794aedf46848b6d9d4b,c293e145-6d05-4dd7-82c6-2c08595ce25f
|
||||
1795180147.213019,warmup,201,0.01767706871032715,211a0e09715649478212fd645fe9c512,1733a12e-bb1c-4f91-a5d5-fb7d8e5254ca
|
||||
1795180147.461827,warmup,201,0.08181881904602051,776ab497422a45e5947ddb531dc82433,8703b6e6-3869-4195-b083-a9bea68a7d37
|
||||
1795180147.704898,warmup,201,0.010509014129638672,7c0b031447754345987cecade08e1e66,68a41089-abbe-463b-a62c-86014f741453
|
||||
1795180147.955076,warmup,201,0.013746023178100586,533292e1cfc5425c80060b0f062c7e74,04cbf080-c35e-4ed5-89a1-02a9696d51a3
|
||||
1795180148.205858,warmup,201,0.025356054306030273,1b139f377f83459caccc028d4b2911c8,77c18782-d7e9-4c28-951e-cc9f4bac2a8b
|
||||
1795180148.4552372,warmup,201,0.01289987564086914,b98cae53f29045d19a8f133e79c9dac9,a0036146-0213-4c84-b15b-d5d8779684b9
|
||||
1795180148.706578,warmup,201,0.02797412872314453,904ee8f5c1d9496bb40cf6d3fa23c656,73824605-99c2-4c59-8b63-82dc61294f03
|
||||
1795180148.963203,warmup,201,0.011016130447387695,5543a1f12ea64c4c9c45d2551f6f8fc5,8e3b04ff-3dfe-411f-894d-5986c58bc61f
|
||||
1795180149.2050078,warmup,201,0.01116323471069336,72ac534ce025497f8d8d1528baf61292,bfdc61c8-accc-4673-a0b7-4523936a42a9
|
||||
1795180149.461789,warmup,201,0.02168107032775879,f42dc751108b4816a1a2776541958326,15b9d117-ce4b-4523-ad0e-2efad6034a24
|
||||
1795180149.710597,warmup,201,0.01814889907836914,d3247bcd8b974042a193c4749779bcc0,509ae033-f080-4c6c-9c22-e26b1ab8c4f9
|
||||
1795180149.9608488,warmup,201,0.015266180038452148,947d80b1b7f24134a13749b0f9054e7c,51a092bb-8aad-4f54-923b-8aeba78e430f
|
||||
1795180150.213044,warmup,201,0.030508041381835938,c2df4f426f204f9cb57c75398cfcd2d6,70a7a0d7-e40e-4f4c-ace2-b9244ddc88e7
|
||||
1795180150.459798,warmup,201,0.015178918838500977,defd7634ee5749bb9cb56938fffb6354,19f848ec-e61a-49ca-a9ab-ee159c377d14
|
||||
1795180150.714624,warmup,201,0.01801013946533203,3b9ebbbbbc004bbbb1173a1decbccb00,68500f45-3a93-4b17-a678-82f4250ebb0c
|
||||
1795180150.964399,warmup,201,0.014775753021240234,d7fdec7d34c5496e96d25c4e8b6f51a2,921a3c63-3103-4882-8383-1e4950642aa4
|
||||
1795180151.214231,warmup,201,0.015651941299438477,a919ca5a526b4cdd91873a89b3249d7f,c75eb90d-4cd7-4419-a1ff-88b6634c5666
|
||||
1795180151.456893,warmup,201,0.01569509506225586,7e65e6891eb749aca2e5cca772c2be51,3528cb80-0103-4249-9063-702d10cbd573
|
||||
1795180151.712977,warmup,201,0.011497974395751953,5f9b3149a64640b79cb47627cfef6355,2d8814ca-6d6d-483a-85ef-5959661861a0
|
||||
1795180151.9644082,warmup,201,0.009419918060302734,c8f9dd041f14439b8e0a88f48c057989,cdf1a975-bc52-4014-b534-59148bf6c73a
|
||||
1795180152.2131011,warmup,201,0.014716863632202148,68761e2b15b64610b364c2a964c6d5a8,3c1cd970-d8f6-4049-9a7e-a574670f1920
|
||||
1795180152.4601622,warmup,201,0.06042981147766113,0ce081bdcc584db8804a460edb6657c1,c0d66583-f650-4120-8f62-3d2c7ae54dfc
|
||||
1795180152.706217,warmup,201,0.020451068878173828,a664c5f673114604aa95b08aa1b7e972,893f49c8-3afa-4766-b58e-c54efa12286a
|
||||
1795180152.9611251,warmup,201,0.025645732879638672,569d9881d33b4e0ebcdb3b55833bdce9,b25ab34a-e7a2-42e7-8674-de4ae680db50
|
||||
1795180153.204326,warmup,201,0.023772239685058594,d15bd802b80a4cf0b5c5b66fe0247211,ba2185ce-cfb1-480b-909e-a9776122877b
|
||||
1795180153.462528,warmup,201,0.04242205619812012,fb3345a7c61d4504b216f861da365b7e,8ffba94b-38cd-4380-a563-8f19403b00b8
|
||||
1795180153.7144508,warmup,201,0.024564027786254883,41c45e295c094077b016ed6bae9a2022,a132391c-bbb2-46b2-b424-bd6a93139ef3
|
||||
1795180153.961816,warmup,201,0.03986001014709473,170e1f3a76bb41fdba945e2e71f6696a,30fba817-b082-4d0e-8ef7-752c23ee4af8
|
||||
1795180154.20493,warmup,201,0.03506898880004883,5bf03ad7556f43718c11e5aa04c57eef,a590d9ea-3b17-4f7e-bd1e-a634d5905445
|
||||
1795180155.218189,warmup,201,0.7187919616699219,2d064afc92574e9dbabc65af8987a7d7,abff9a4d-707b-4626-96c5-698361062611
|
||||
1795180154.4665148,warmup,201,1.4753811359405518,670d7efd7b35456ba9b976db307daa78,1a183e3a-1b15-4d3d-94e1-74c22a1d02fc
|
||||
1795180155.711974,warmup,201,0.2379162311553955,5bb3401f86f246dd8ee36d2e60e31858,1d37a14b-850d-469d-8b2f-8cf48b9c1c24
|
||||
1795180155.0022368,warmup,201,0.965947151184082,ae126ae644014ec5b1936e416ed2a435,e52afb5e-8df8-4ed8-912e-d83a28d52c9e
|
||||
1795180154.709724,warmup,201,1.354740858078003,e77df7dd28f84c3d95a7cc15d5e98456,d8df3654-4ccd-4a1a-a4fe-d747ed6bb174
|
||||
1795180155.473359,warmup,201,0.5913128852844238,fa2ad4a0d89f44c58b539416eb37b3b8,01cce25d-695e-455a-9b89-3277459c6471
|
||||
1795180155.963171,warmup,201,0.10228514671325684,a90640776b8c48d38e0ec681f22ac39a,0301e0fa-5b43-4ec3-bde6-339af08a4e2a
|
||||
1795180156.214087,warmup,201,0.018687963485717773,7f967f771f324b2da7abe59c4daef1dc,1beda72b-b70d-41ba-990c-e5a062595310
|
||||
1795180156.462667,warmup,201,0.018568992614746094,737f8ea6e6d84bd3b51cffe92b59a69c,53dfeaeb-b8bf-44b0-b526-5b16bff819e9
|
||||
1795180156.7076569,warmup,201,0.02150106430053711,759761e721b142b9a418803b6df9c027,a06894db-6c95-46d0-8e17-60b650716376
|
||||
1795180156.963069,warmup,201,0.020072221755981445,73a497e1f8cb490fabd21547c58e94c4,b008b068-3ff8-4bcc-8baa-b2aa568e0628
|
||||
1795180157.210342,warmup,201,0.02046990394592285,8779184ae9ba4e23a3fc2a99f6980866,a15fd300-6132-4d3e-b1de-7264c7edb3f6
|
||||
1795180157.4589682,warmup,201,0.032640933990478516,60829f4073814d08a8aebc311277ff33,82e61037-01df-4764-ac06-21e048a1596d
|
||||
1795180157.713058,warmup,201,0.04641890525817871,84530fe1b329459ba8c05fbecc725d96,c23fb1ac-67fd-48f5-924d-6fad526fc69c
|
||||
1795180157.960186,warmup,201,0.0545649528503418,b6e31f7e82314adebd2c96abb12976f6,00b781e8-1a6e-4d1a-ae34-75d2a1990179
|
||||
1795180158.2074811,warmup,201,0.024766921997070312,2985a09493b1491c97cae490772bfc8f,d02ce7b2-e0a1-439c-8a02-aa57ad954b0e
|
||||
1795180158.4595351,warmup,201,0.1045236587524414,921a1935688f49e5ab1828cd40a3c375,fd12108f-603a-44d5-8d93-d29675978d04
|
||||
1795180158.987233,warmup,201,0.26446008682250977,93913d95e58246299d872da8dedd7920,5aee7bc6-a089-44af-9753-a647806fa999
|
||||
1795180158.708724,warmup,201,0.6227638721466064,7b8b3b12ecde4fbcbeb074dff21b663b,cdb7a2a2-2392-4bb6-8feb-18f6c37c22cf
|
||||
1795180159.221353,warmup,201,0.20409274101257324,2a411aca603948868e5b73cda6fe91e5,83299336-71c3-4f85-afd5-9a7d632903be
|
||||
1795180159.463039,warmup,201,0.09139418601989746,26b02c0f71f24e5c8ddc96b9d1ae25a5,c32941b6-a35e-4780-91ed-17cec0715c14
|
||||
1795180159.706288,warmup,201,0.028244972229003906,619307cbe4bf417da6c149d3f3cb5b8c,a5fa4e6c-2878-4e01-82b2-678857cfdfa8
|
||||
1795180159.956794,warmup,201,0.012042045593261719,bb655577487a4fc8ace4f902c8e8c199,5ede45e8-2c45-4a1b-a9bd-6f04fdb58c97
|
||||
1795180160.213904,warmup,201,0.05432319641113281,8f7670e1d09241e8937e71bcd4b108d3,f9e8358d-0216-4351-a4cf-8a1b57e1b515
|
||||
1795180160.463031,warmup,201,0.07005786895751953,81fe6f93c621412393a7521026fb86dc,de99a316-38a4-4017-af7e-e10e6a4116b0
|
||||
1795180160.705324,warmup,201,0.05199694633483887,7e9577a3bf80456a900e1745534ab5e4,22e6878b-31a2-4364-a94a-54f9daa82365
|
||||
1795180160.956168,warmup,201,0.054311275482177734,88a73eeb3a41494688161931b3f2d0f2,dce46916-4572-4df2-9bdf-70a7b940cd2a
|
||||
1795180161.206967,warmup,201,0.08410191535949707,acf2f4457b724ab5b3da4ab0f57a4c3d,09ef6df9-78f2-4502-80ca-672166481b98
|
||||
1795180161.457957,warmup,201,0.03756880760192871,eeb99aed3aa2474682bb942249b73f96,de89ee75-a4a0-4f50-b7a8-e305cf43126d
|
||||
1795180161.707008,warmup,201,0.023017168045043945,a9263d7d93d04c818df6021d471c3fff,3ec8e5f8-2f11-4cf7-a965-337078d571db
|
||||
1795180161.954315,warmup,201,0.03162097930908203,24367be67ee24f9dabb51672dbc58e04,25172e43-db3c-49a3-8294-187ebc18c469
|
||||
1795180162.207732,warmup,201,0.02358698844909668,de045035ddbc4759a8ff1e8c03e62657,819465ee-7403-43e3-ab30-10db4551195a
|
||||
1795180162.4623191,warmup,201,0.013826847076416016,bed4c0d33e864f52a18dae225d33ed50,b13abc97-d995-4b5c-9749-4e85892946c0
|
||||
1795180162.7049441,warmup,201,0.011426925659179688,37bcbf41ff8a48939d46728863884410,90f819ad-45b1-4e79-b583-45c26033f52e
|
||||
1795180162.95575,warmup,201,0.013515949249267578,f7c1e2922ab84f149cc615144d947801,e9f054a5-969b-4714-b3e6-948247656538
|
||||
1795180163.2059221,warmup,201,0.014194965362548828,7e0523de687c4c358f88f591eb45631d,d708f6f8-6747-4a1c-82a8-7056048525a4
|
||||
1795180163.457537,warmup,201,0.012165069580078125,248b9bec4fe34a5d991fd6246acf0c22,13bff701-eab6-4c76-aebb-1b4fe929d1b2
|
||||
1795180163.704688,warmup,201,0.010187864303588867,73f64cbad982461cb68e042fb8b2ef6e,5247ee5c-efad-4a2e-87cf-e955a9c39ce1
|
||||
1795180163.955895,warmup,201,0.01818704605102539,c6c1a0040a2f40d1b729f6e0895f15e8,e5a1c9b2-68fd-4c1f-aeae-4450649c1556
|
||||
1795180164.20519,warmup,201,0.01907491683959961,00e6a3e5c771484dbb237cae0fbfc09d,bc2c0346-9643-4d27-a6f7-689b1bed3e85
|
||||
1795180164.4559622,warmup,201,0.02111077308654785,4665df73d5364e5daa3a441143b7c127,a8594acb-4d13-4686-b53b-0346cb860fd5
|
||||
1795180164.713967,warmup,201,0.014435768127441406,e1c3c6397f7f428b85cbcacdb1098b80,b07823c9-ee89-4dbc-ade8-321d3974dc56
|
||||
1795180164.96303,warmup,201,0.021161794662475586,26d86cd296754a089f91b90439e83f60,8ac96c25-a31d-408e-a3fa-b2e6f372a454
|
||||
1795180165.207382,warmup,201,0.025116920471191406,09994b987d8e47898ddad4fb10a09a38,e0d41a3f-d634-42c5-9496-bbecbbe7d733
|
||||
1795180165.463873,warmup,201,0.015794992446899414,df1d6019189446f09582a5bc32757299,7f49017e-5ac2-4521-9772-74d2e55ff1d9
|
||||
1795180165.7129421,warmup,201,0.012481927871704102,cfabbf97b3c14b02bfba1a74acaf4310,add4a145-d07a-49d7-8cc1-5f9774f6d5bf
|
||||
1795180165.961084,warmup,201,0.019973039627075195,5b48ac86de57401eb7f3991cc1904749,7a498621-f882-474a-89a8-486990c9ed2e
|
||||
1795180166.2129421,warmup,201,0.010935068130493164,803f2dd4c81146fd9a5d460b58a5256a,f20ae48e-27d6-476e-8207-e99a9e68019a
|
||||
1795180166.454031,warmup,201,0.030212879180908203,b40cfbdfebc044e3b29b3621d95f536a,3df1f455-41e5-4aa7-bfef-8754f1eda936
|
||||
1795180166.712905,warmup,201,0.02558112144470215,00b1bf8353934579a8a95b922f1fdabe,ed4d2d9f-b932-40a3-85e5-a9dc6a613e9e
|
||||
1795180166.9541728,warmup,201,0.014276266098022461,9425018ce8324791b8d6eff29d7ee14b,9cfd6c36-ba55-43a4-9d51-13273a3a80df
|
||||
1795180167.213747,warmup,201,0.08199000358581543,b4c47ffe312b4aac87827583c0843fde,a414a75c-0936-42af-8715-7013f1e94527
|
||||
1795180167.4605298,warmup,201,0.015784263610839844,84d6ce144a2c40cb95f5ac7690fac5d3,1b856fcc-ff52-4147-90e1-cd94791f7941
|
||||
1795180167.7135592,warmup,201,0.026343822479248047,b7939325d6c34ecea0d35c5d744d3bad,6b28b39b-0f68-4b90-a51f-1c152c58a6c6
|
||||
1795180167.962919,warmup,201,0.015112161636352539,cd5cb0616a7743448c76ce7c9d9b314c,72dfdf21-f5e6-4b74-a1a0-e977ed622969
|
||||
1795180168.2039711,warmup,201,0.011447668075561523,1166a0cbf6474c999930bda4cddd6116,02ab9f26-5ba3-4201-9c58-15aa457967f7
|
||||
1795180168.458719,warmup,201,0.013019084930419922,cbac190c2cc744c1bef9610fec68ddbe,c762f7bd-4ea6-4f4a-a59b-fcfa20f2d405
|
||||
1795180168.712939,warmup,201,0.03552412986755371,f4f37c03503344db9ab375351a65cd3e,255cef6f-545f-4ae6-ad6e-99ab8807e772
|
||||
1795180168.957678,warmup,201,0.043527841567993164,b17d2bb8e0714a2d901e1f18637a2814,083f8307-1763-4a1d-b309-0d8d702e5060
|
||||
1795180169.212994,warmup,201,0.0400240421295166,69ac03cd26df493790cfe98a041d5ef6,d0cbec89-cc67-4617-a0d0-26dddb89eae5
|
||||
1795180169.462979,warmup,201,0.01599907875061035,2790773654d84f8d9887e150c9a7d7b3,e67158c3-4809-41f3-b959-9f8c8b06ead7
|
||||
1795180169.712948,warmup,201,0.0437169075012207,4c0ffa1b3d414b4cb5c22e9ce7383a56,58d0c598-8d58-404c-9cea-d3a109fa3886
|
||||
1795180169.9544451,warmup,201,0.20522069931030273,a5a0cfd85436458187897f6cfb92e828,4113d976-439f-4909-9baa-f550c1991184
|
||||
1795180170.2053828,warmup,201,0.044216156005859375,f94451b87409492ba02e70fa8f3877c9,9c45f131-93ff-4678-9ac9-1857ea5116e8
|
||||
1795180170.456413,warmup,201,0.028231143951416016,40ee2a373b2943918e5d19b44780f7c2,b8d0315e-40ff-4b20-a7c4-10eb8a89abd8
|
||||
1795180170.705407,warmup,201,0.14519500732421875,3fe37ad54f6449628ea8ae3b55fabda5,8c124926-7f46-491b-9bf6-9155a58e33c1
|
||||
1795180170.957385,warmup,201,0.1566169261932373,bd234bbdd7a84eef91106d8bb06e4d82,43c16843-8556-4fcf-a412-7c9c85de1829
|
||||
1795180171.209719,warmup,201,0.02808213233947754,1bc7c13575584cc3a996d8e7d4e7130c,f59677d7-0cd4-45d7-8bba-4843b1a3b675
|
||||
1795180171.708301,warmup,201,0.03726792335510254,eb4cff5c04cd476db79a908172e69ea4,26149134-587c-4036-8f48-f7e18ef4fe50
|
||||
1795180171.464971,warmup,201,0.3508949279785156,a9a6b04df882480c9bacfdc2a07216b8,5f0fd18b-a430-4f50-9076-e8f8ab1f047a
|
||||
1795180172.206183,warmup,201,0.16024208068847656,7855481b6d0541a087d7f15273a8b09a,247ddeec-8837-4c1d-a60a-2546978fd953
|
||||
1795180171.957116,warmup,201,0.6614320278167725,0963b0b6357f44278d2c3cdfb17e4250,693d8bf9-8e2e-4efc-93ee-be9cbac0b8b4
|
||||
1795180172.4628391,warmup,201,0.2640187740325928,18253581a2bd495aa28106a53393968f,c41b1cfa-02d9-43b5-b055-f014f7f57802
|
||||
1795180172.704912,warmup,201,0.03334403038024902,e3a735c71221445fa876a3e44c9cfa47,2ccac5ea-d3ab-4bbb-b063-3b7c36c3ea4e
|
||||
1795180172.9577332,warmup,201,0.06759285926818848,af4bb1940dbe42538d232d83ca9812cc,cf812735-2489-4e04-b672-b6143c2c78d0
|
||||
1795180173.206496,warmup,201,0.024734973907470703,70f1582dba2f421ea1980dd833dd5a9d,22898e79-875a-4898-a300-845aa4a6d199
|
||||
1795180173.455349,warmup,201,0.03874802589416504,e858bf8007064349bca1e03aab5eb26f,27cc175e-827a-4188-a728-b78cbc3967c9
|
||||
1795180173.7049768,warmup,201,0.0689091682434082,a066bdd03a4043f3ad62d43d306acdaa,c07f5205-97d4-4f59-a9aa-8d05bfc1a52c
|
||||
1795180173.9543931,warmup,201,0.018208980560302734,2e50d32ac368413cb7c4f82c97b55bf1,be9067a9-5d8d-4efb-bb8b-8077e902078e
|
||||
1795180174.2129998,warmup,201,0.012910127639770508,2298a50853b442cf908afff6cd410dda,51aa7305-711e-4137-90b3-34a233cd9587
|
||||
1795180174.46289,warmup,201,0.011858940124511719,fa1e496a8fa54b3bb1168cce8e1a4f23,9f1088f8-a141-4091-a877-fc8d93834d91
|
||||
1795180174.708926,warmup,201,0.013499021530151367,99f56b141a6d499aab6492480f6fa0df,d5705025-1ab9-4976-8af8-bec4a632f7f6
|
||||
1795180174.961039,warmup,201,0.05283308029174805,c446fb3191384b2fb20905a33ca1259d,7b9d27c2-7bc5-4768-bbb2-7b433f668401
|
||||
1795180175.2130811,warmup,201,0.046158790588378906,6cac815449fc43ba9ee764b2d3ea89ea,b44e4411-77d1-4acb-a6f2-f94fb6b639a1
|
||||
1795180175.459479,warmup,201,0.08919286727905273,00672bb9be9641269cac74d9054f8f7c,27f04e7e-a0fd-4f67-b93b-32560341c85b
|
||||
1795180175.713186,warmup,201,0.07668614387512207,909316d4377046908d3d9b80c5f63734,6c14d788-4b33-4886-8fcc-b87c34433962
|
||||
1795180175.95504,warmup,201,0.02902507781982422,e11701b75c89461f9cf9eb4c49119993,34580b41-aa71-4bf8-935e-4b661f85326e
|
||||
1795180176.2096431,warmup,201,0.023485898971557617,94dca22c54ce4adc9773397279aeeb94,4778e745-e0dd-4ba9-9739-23f228358723
|
||||
1795180176.454023,warmup,201,0.013267993927001953,0c731c2d3c904251a1aaddc02936ac72,b31feb95-00d0-4633-bef1-d3a80d59e132
|
||||
1795180176.712258,warmup,201,0.00846099853515625,5ef7d7287a94477fa7506a8974c8e073,9e76b067-3773-4042-b008-3bcc8ae37289
|
||||
1795180176.9543521,warmup,201,0.008093833923339844,fc63470f48ff40cca32ea6bbd1c87df6,0ef443fe-2d88-4098-828d-5dcd08f0cde5
|
||||
1795180177.204947,warmup,201,0.010506868362426758,f302b68d4cb641eba2ade0caa0d5e7a5,d11c40b9-87d3-4c6c-942d-f0cfb14c72a8
|
||||
1795180177.462801,warmup,201,0.014312028884887695,d62ebff655b64a349542bae704bc43c3,6a603f9c-232b-4fb7-86d2-4d762a4e6e89
|
||||
1795180177.70669,warmup,201,0.011358976364135742,b76a61a69f024f979a61e9ac4206ba23,d252996f-f60b-436b-88e2-5f3f5a19774f
|
||||
1795180177.958333,warmup,201,0.010645866394042969,f81e9d7fe46e4d3f9d2c629a64cf0532,70b74b69-55d0-4b4e-8a63-09f3ca44988b
|
||||
1795180178.21154,warmup,201,0.011842012405395508,d927e2354c124bb1b133252d7ca78ec0,59f1a68a-dc24-4ec1-a940-18e8c75c5544
|
||||
1795180178.456432,warmup,201,0.016075849533081055,fab54f7549c84c32a9333b523bbaf313,4c301fd0-c4ef-41b8-b803-2d0f3d4a47fa
|
||||
1795180178.705439,warmup,201,0.016112804412841797,6795365d631349b8a2b6b0f020fe56d8,f60cab03-ca55-4f74-a133-c5c8ed18c7e3
|
||||
1795180178.957304,warmup,201,0.012420892715454102,76ce4d9cb2d347e7a54f347e1555059e,525440d2-b4bf-4250-9b7b-31e89fb57a27
|
||||
1795180179.21025,warmup,201,0.008681058883666992,d6b683ecea3741df86168a9ca995ea9c,cd03fb05-c3b4-4b57-8151-c5e749cfe2c9
|
||||
1795180179.461452,warmup,201,0.015002965927124023,cba9638961a040e3bcb4370420e5a858,f7facafa-db9b-4922-9dc9-e65500737f75
|
||||
1795180179.712847,warmup,201,0.015548944473266602,bf553ad417b040419ae155120de940bc,2aea59d2-0e2f-4929-9651-43073423520d
|
||||
1795180179.960278,warmup,201,0.01565384864807129,ef20979209bb4e448255f580b1a07b61,c20f1a67-e981-4473-b1e0-422fd88447e7
|
||||
1795180180.213103,warmup,201,0.015707015991210938,a32354c47bd845f5bfeb45c98f03c848,9334f89b-289f-4d50-9b87-0afd45cd05aa
|
||||
1795180180.459705,warmup,201,0.012166976928710938,f0fcbc4e2d4b41f098e3429bb23ca236,f856a16e-3469-494a-afd7-943339c4956c
|
||||
1795180180.713276,warmup,201,0.019527196884155273,93e6ed25d7654a8a8aa5a6191c03ae82,a5c347ed-5b9d-4df2-836a-29ddb2d05126
|
||||
1795180180.95435,warmup,201,0.016804933547973633,edae63a4317c42b4a2afde5303f1871a,efbff025-41ee-4761-bf4d-3dbe23d4007f
|
||||
1795180181.203885,warmup,201,0.015861034393310547,4531a8464acb49d789e97b7d2cd44f74,79d25593-3116-4f7e-b555-01dacf572512
|
||||
1795180181.4541068,warmup,201,0.010253190994262695,b35a9cb2393442ec88077c5277b69223,cb921d6a-632f-4966-9259-46b8ff7a936b
|
||||
1795180181.712745,warmup,201,0.04211616516113281,b55996e8ad384e92bd22553058887531,0befa854-1659-4b69-82ee-2d9c4452c56e
|
||||
1795180181.962806,warmup,201,0.014417886734008789,eb54c368418c4ea2a81cc55c18f33674,d1339fe5-4fcf-4c51-aa31-ca0d96a21846
|
||||
1795180182.204462,warmup,201,0.012550830841064453,27545433b2c64c5585b1f857b04c41af,2cae5a6a-1c7a-4f5a-92e8-e31936a8deaa
|
||||
1795180182.462759,warmup,201,0.013756036758422852,ba01f205cd36452f91acdcc144693ba3,3ca8359a-17f4-4a02-809a-a0b3258c53bf
|
||||
1795180182.7060359,warmup,201,0.016193151473999023,c9446e7e7bbd4435ac4dd80d99da6237,18f5e9a2-e710-4942-80d9-55604faeef13
|
||||
1795180182.956501,warmup,201,0.022050142288208008,5c19e1f85a624e03a79ed1bb9d338b27,0aeb8bc4-2242-449a-a655-4cbd016f1ee1
|
||||
1795180183.2053308,warmup,201,0.014279365539550781,b35b8690a19f44459906848fc8ea13fb,53bb3ff2-cd49-467c-b323-73ae30e5dafb
|
||||
1795180183.462892,warmup,201,0.05702996253967285,929910e78a024ae092c58d12a1852b24,14309d64-5f24-4db6-b641-3c5e031e1204
|
||||
1795180183.706051,warmup,201,0.03141498565673828,a4e93ec239574d57b20f10bd5e7560ad,b6771803-b769-4d57-8b38-1ed6fc6eab85
|
||||
1795180183.960104,warmup,201,0.19160008430480957,646d23e10c5b44f494eb4538753c608b,1bf69a09-4231-4a27-9bfc-6f728053852f
|
||||
1795180184.203552,warmup,201,0.04222893714904785,577e8b16c9cb49b4828a8dd043e749ec,5e8e8000-b2db-4d94-b1c8-0a6636d678f7
|
||||
1795180184.454276,warmup,201,0.014716863632202148,27fd975e87d24480ad32d1ff15d95cae,24f15864-5d6d-4d0c-aed0-8afba06f22b3
|
||||
1795180184.707995,warmup,201,0.03830385208129883,8c2abf5443824c13b1d16a6b7ab8fefd,2cf6a6cc-2a28-4a5b-8c77-7228fde62e7c
|
||||
1795180184.963094,warmup,201,0.01816391944885254,4db1a0d088e549f6ad9b3fe4e3bfca7b,8d0d23ad-6b5b-49f8-a967-0f81fbf04fa4
|
||||
1795180185.213154,warmup,201,0.011205911636352539,277ddd9fe9bd4c798830e9f0b690aa58,940bacab-a245-4c62-85dd-cf7dc89773bf
|
||||
1795180185.458055,warmup,201,0.01757192611694336,89a9630e6b3f4accaa02a5e4fcbfa9b2,0e2796e4-eea9-441e-ba7a-adc3318c4bcd
|
||||
1795180185.717603,warmup,201,0.04112505912780762,0c81fd8100a644d886ab797b31454edf,2a6d1be5-813e-4431-8acb-de28bd6cc485
|
||||
1795180185.9632158,warmup,201,0.014150142669677734,f726becda74545468eccdc64fe022e3c,3310626b-3ff1-49f9-85a3-d46b310862ec
|
||||
1795180186.2084239,warmup,201,0.013811111450195312,ea4985ca5f3340f1878f7790e1d7dee3,bb9f5be6-eee7-4fae-9d7d-708400a5f332
|
||||
1795180186.454124,warmup,201,0.027569055557250977,02b44be192d646bd94c431a7e0cb6ce3,d57d83f0-2dae-4fc2-a1ef-988fe1f370f4
|
||||
1795180186.713099,warmup,201,0.011459112167358398,0dd312b7244a4d5887379d5c0a002e66,6a1c1ae8-f2ee-4851-9b63-6180c5b7b7ec
|
||||
1795180186.956599,warmup,201,0.019530057907104492,92341c3304f3480ca74f390353e35541,54f84857-de49-4ba2-a228-0737e5c4519a
|
||||
1795180187.208894,warmup,201,0.12175798416137695,624595d09b0b4341a6ee3562a5acedc3,c9526170-ffc5-4155-be96-f4afaf01ab28
|
||||
1795180187.4532921,warmup,201,0.030305862426757812,76f2be0fd53c43929d5dbd13c8ea9f4f,a14968b7-2b52-4296-8a57-dc2da3cde3e9
|
||||
1795180187.7090461,warmup,201,0.021664857864379883,b8aa6f23adba449cb738ddb921c834bf,cc87f9ee-79d8-4bee-8afa-6d9bdc0e50a0
|
||||
1795180187.956699,warmup,201,0.011160135269165039,fb8bc2ed80e248a180abdaac7d680b97,c67baf4c-da95-4b1c-8fd9-54759dd87f7e
|
||||
1795180188.203029,warmup,201,0.014909982681274414,6b91e855e5004849aaa0d0b611cf98f7,44162c07-2165-4353-acea-cbdb32f1f389
|
||||
1795180188.4538748,warmup,201,0.04066634178161621,94c834f81c084b91938fab2d21851366,fb7693b5-7fbc-4675-92cd-3952ec2ecb97
|
||||
1795180188.704454,warmup,201,0.018157005310058594,d759a840803945fe9a93443f2fdd0b2e,4444f578-97a4-4aa9-82c9-a1bd51b35dd7
|
||||
1795180188.963008,warmup,201,0.013981103897094727,f57660a0e9604d7a825a6f5e3743bd94,8577c34d-68b4-4ffb-93ac-4b7bd4c83f21
|
||||
1795180189.203331,warmup,201,0.023202896118164062,244114a321374316adc578862f9200c5,1323a0aa-0302-4608-87b8-55451b318c97
|
||||
1795180189.460938,warmup,201,0.015442848205566406,693f36b531444b85a5ada1bae9069a5d,65fc0dab-bb6b-40e6-9d93-218728ef2118
|
||||
1795180189.7105951,warmup,201,0.010252714157104492,775511c278684cffb821bfdbdbb40ff1,082dea81-4649-43da-af39-8148f5e3e17f
|
||||
1795180189.962133,warmup,201,0.027779102325439453,704f62a681bb4d1d902eeecc72dc8c49,020729eb-6c61-4562-ab6e-4261cbb48cb7
|
||||
1795180190.208668,warmup,201,0.08618903160095215,d58246f4898b413cb3a57efecf51996e,182f947a-cffb-472b-88e4-ae63b9055269
|
||||
1795180190.4556952,warmup,201,0.012355804443359375,85c0ca16930f4f209da3a945ce5c4c0d,48f24757-6028-41af-b1d7-8a4f16f0c467
|
||||
1795180190.712575,warmup,201,0.021033048629760742,8b5ce8b40b5c4735adb4be19e104a00e,75758c4f-8b76-49a1-bf16-396c6d6f074f
|
||||
1795180190.9550889,warmup,201,0.01253509521484375,20981200b4c649a2b3eb03ea7d2953ff,259fa757-9ecc-4169-a160-c9c5b765a199
|
||||
1795180191.212717,warmup,201,0.009023904800415039,18e3ddcd29164515b49ae4df4546ece9,9cd61279-0edd-4c2b-87ad-7da34a443709
|
||||
1795180191.462502,warmup,201,0.01763176918029785,e28e7f1f48144770af9dcffb74421210,272f31c8-dd14-448c-a2b0-0232ead05ddb
|
||||
1795180191.711782,warmup,201,0.015043973922729492,a887526046214c9296d8d7abe36313c9,004511dd-d814-4b80-9979-1e68dfc12f8f
|
||||
1795180191.963177,warmup,201,0.020080089569091797,6e0ffd38e2224d7fa79adfe0bc392818,900a372f-3281-4637-a8d7-8273d340d0f6
|
||||
1795180192.21304,warmup,201,0.03302407264709473,d3b15ced1f164a9eb8b0af6e883a5d0c,5ef64385-53b9-4f95-ae89-866084089856
|
||||
1795180192.4593432,warmup,201,0.026756763458251953,288161a679294cf991c8cf1f700bbee8,5631d6b2-9b07-4d05-be87-8dc2b82dd2a5
|
||||
1795180192.706445,warmup,201,0.02288222312927246,6ff73f860a2d47ada41da4ef9b9d8c1a,11733eb0-f5ef-4e7b-8667-092abf95794a
|
||||
1795180192.962646,warmup,201,0.021658897399902344,d12c7188544d47e6ac8491fab039b363,ac8ad5fd-c7c8-4373-b22b-9a8630e607da
|
||||
1795180193.2061381,warmup,201,0.019336938858032227,075af4e320e344da911d37c4c5a34a91,d54c1b25-df52-4413-a8bf-c00e8fd67c7b
|
||||
1795180193.458136,warmup,201,0.017496109008789062,8c165225b2d64d238b3ad2bd4a696c25,e7fb5585-af0d-41f4-a0b9-a14066005edb
|
||||
1795180193.705204,warmup,201,0.022547006607055664,9599bf2b238a4c78b7a8fda52bc67ff5,e7bb66f1-582a-45c1-b29d-aa4dfcf01619
|
||||
1795180193.9539452,warmup,201,0.26253581047058105,cf4db02bb778433f9ee50fa7fa47cfd5,cc18753c-758e-4be2-b511-9ba6da8872d0
|
||||
1795180194.2069068,warmup,201,0.022910356521606445,cae9dd0a6bf442a4b8e67bad0875c97b,958e8fd7-07f0-4812-8871-ef48157cf903
|
||||
1795180194.460042,warmup,201,0.25853991508483887,885dd8b642104c1fa04f84688d816fc9,f9b1e565-f8e6-4abc-9e7a-65a1c9e4a40c
|
||||
1795180194.7131689,warmup,201,0.07060122489929199,1c56908d205741b2964f718ddb5f3def,69258830-947c-4245-a833-78176422fe70
|
||||
1795180194.952873,warmup,201,0.2982909679412842,8ac2ceb1f42040f8bc6e894fe13abbdc,3dad5269-9754-454e-b9cd-264f86774850
|
||||
1795180195.4555662,warmup,201,0.05453991889953613,3592f8a05103400fa96fcd57e0f5f7c1,c078f8f6-f584-43fd-85df-acabc6b876a9
|
||||
1795180195.207063,warmup,201,0.950869083404541,80b7746064cb43a2adc4f588a84601e5,f0b4607e-5350-45ee-b07f-2be09e95fd99
|
||||
1795180195.70959,warmup,201,0.4569091796875,7b2995e04de8452db02d42aed9eaca94,4cda4fbd-ab10-48bd-88ca-3c48af14315a
|
||||
1795180195.956768,warmup,201,0.22559404373168945,780d3f933edf4e08be22421f5cbb4162,39d0a6bf-c992-4cd2-b4e5-f7796a042a6b
|
||||
1795180196.209187,warmup,201,0.05438494682312012,f5ea813035874d0fb0b08e826ead91e0,fea354b4-7a1c-496c-a804-5e572c0767f3
|
||||
1795180196.454772,warmup,201,0.04239296913146973,bd79b9a297a34dc49a4d6a3b1cba46c2,6a63f12d-a5a1-47d2-bf37-813c326442e3
|
||||
1795180196.706481,warmup,201,0.02643108367919922,615b66824e97490bba5193a10df8fb5e,ccc105e9-ebbd-4f96-ad46-85d2ae34e259
|
||||
1795180196.953847,warmup,201,0.024856090545654297,1ddc5ff0885040069f2ebfcbe03bb60c,5d94b5f8-2496-4f7f-8fe2-23c4975a3874
|
||||
1795180197.21262,warmup,201,0.015910863876342773,609bb8bdd431457186d75aac72c48f86,9b98b6b1-31ce-4825-b220-c92fc034d556
|
||||
1795180197.455989,warmup,201,0.03686928749084473,f91f41fc62b54c05b486321f29266270,0c44435a-4ede-4ee1-b83b-4f119384dcaa
|
||||
1795180197.703308,warmup,201,0.02806401252746582,42ecb8cd11ef4e60b95a28d839f19903,8eaf59f5-d4b1-4079-9162-87292d73f31e
|
||||
1795180197.962825,warmup,201,0.012208938598632812,2411d19e7ea942ebbe8d97ceec71bc0b,80a79fa4-e59c-47c1-9048-5cba540a330b
|
||||
1795180198.2126641,warmup,201,0.013143777847290039,faac45e8c35b4e22907ec7a29991e111,f9de74b6-00a9-4736-b224-e50f0e07fca8
|
||||
1795180198.452794,warmup,201,0.010195016860961914,216c854ded9b40ad84c29e4dca3fb40e,8b8b2144-eeb7-4bfb-bcf6-1c3fbd1d3500
|
||||
1795180198.710228,warmup,201,0.01584315299987793,bb930213cbeb48f4be1c535416c3ebef,63588c1d-2d95-4935-9daf-4b6591f718ac
|
||||
1795180198.955477,warmup,201,0.022932052612304688,98735988ba83477c8dc8255e39c6e9be,4f26cb6e-f5e0-43f5-9ff3-00410d4fcb0d
|
||||
1795180199.20573,warmup,201,0.027022123336791992,84770b0f61c3443b9c5b95d8dca08be8,44c5c309-221e-4296-a57b-5a2624514949
|
||||
1795180199.455035,warmup,201,0.0162200927734375,a13929fa98d745b6badf08b6afed734f,616a1684-6a86-455f-9c35-98d63f88a237
|
||||
1795180199.7043462,warmup,201,0.021170854568481445,1a22338792844a6995025059d1573d8b,3fd685cb-bb6c-4b53-a2f2-e67d66e52ecf
|
||||
1795180199.9531882,warmup,201,0.013972997665405273,1803c2fcc5644cd5862fa1c5ba572c5f,ada74ea3-81f8-40b1-bb49-62b850483cb6
|
||||
1795180200.212534,warmup,201,0.016909122467041016,f0eb8bba585640e59b164fa58243df6a,95fd836a-e610-40f7-bded-88cf8bbc4d9e
|
||||
1795180200.4543731,warmup,201,0.011936664581298828,cb40183ca2e84009bb204da20c980af0,172837b0-53d7-4056-a9a0-6034f129a3fa
|
||||
1795180200.7096949,warmup,201,0.0234220027923584,399c3e459be045dfb2bae117f1ce2dbd,93cd6edc-c047-4411-befb-6e07e178e52f
|
||||
1795180200.954174,warmup,201,0.02118086814880371,c9fa117ad07a462d84c7b43f8922716a,f5fd669c-648f-494b-b836-02a6e00fc0dd
|
||||
1795180201.203704,warmup,201,0.02590489387512207,b23bb2e856984c60b6c634c3f5a1bf8e,231408c4-a640-46ec-b694-ba7b7b26808d
|
||||
1795180201.4537852,warmup,201,0.012441873550415039,293dfa83cfdd475ea7466ed8741d856c,eec778a0-e94c-4fc8-960f-98d89e79160a
|
||||
1795180201.7075672,warmup,201,0.009174585342407227,46c8ada6c78149efa0a7ca8e4a5116c2,cdfd8d19-fb43-49dd-8a9f-2ebfec681b55
|
||||
1795180201.953985,warmup,201,0.014340877532958984,fab2d1b470a04bd59bbd3ab89449a39c,5338f15e-4c79-428d-aed7-abb03a0d2e38
|
||||
1795180202.212538,warmup,201,0.024313926696777344,53fac7e2df3e480db30b3d7249691ad2,04d13d7d-12fe-44ec-9185-84397ec920e3
|
||||
1795180202.454725,warmup,201,0.011285781860351562,77d6d1070dac42308a797d848af7d655,7f7e9623-4f82-4732-913b-f5ef7419906c
|
||||
1795180202.704545,warmup,201,0.01698780059814453,d1eca4e404474f548e4c9433975ff7e1,37f34032-000f-497f-9a2d-af4edea502b9
|
||||
1795180202.955153,warmup,201,0.012134075164794922,b3ba6e15f8464b80a44e6b201195a5ce,98fa1948-dcff-4dca-86e4-028ee0edba3d
|
||||
1795180203.208073,warmup,201,0.04192614555358887,a3100e87452b4ea6aeb47bf91637d1a7,d334cda7-3c90-4805-97e8-64b8e2f70a1b
|
||||
1795180203.4534369,warmup,201,0.019451141357421875,42366783be314ea1a699ee7b04915272,882e0ea6-0a04-47d8-84c9-535ff6ba8cb7
|
||||
1795180203.705002,warmup,201,0.019451141357421875,cbe35d7bf653422fbed291fdac71b9d8,fbad4794-3478-4d93-b051-1a7e7aa973ae
|
||||
1795180203.9624739,warmup,201,0.020875215530395508,ae17f67d661349cc8def40a8ef9c50ef,78d6450e-c4ae-42b7-b8d0-e84b45063417
|
||||
1795180204.206752,warmup,201,0.013983011245727539,4cc9c8169d364dce9be1b8f554f1b904,a22865ce-7d25-4d11-80c7-eaab1be90dad
|
||||
1795180204.462346,warmup,201,0.01632404327392578,e7fa08a3cfce42e590897ab29d406ed9,f0a8ceb3-5b07-4e6d-af9e-b268b1ae4f6b
|
||||
1795180204.702373,warmup,201,0.027428865432739258,7fe94400e2124e1f994f4ceee5a86b61,fe1fd31b-25e5-4426-abe7-47d20a216f68
|
||||
1795180204.9624069,warmup,201,0.022263050079345703,1e442c69b70d4c91a2367e58d7c85152,410322ca-61b6-46ad-a236-3403bfd0304f
|
||||
1795180205.2122922,warmup,201,0.00986170768737793,e38450ee6a9843288edccd96a9073daa,d2102740-3b08-47cb-bf40-a3c5a083d68d
|
||||
1795180205.4595919,warmup,201,0.012141227722167969,3549a7e31a724d64a9838531e9858237,94382e93-ef4b-4521-b721-fd16cd9a0aa5
|
||||
1795180205.7061708,warmup,201,0.06794524192810059,c7972779afde4c389d99a40a0a1ade6e,7da588d3-2ad5-4b84-a5e2-c5bb86179706
|
||||
1795180205.960969,warmup,201,0.017190217971801758,421aad87ea14472d9fbda6c3dd9ba7d7,4e603bc5-dcb7-4a97-971c-d11de4f95acb
|
||||
1795180206.2061582,warmup,201,0.018009662628173828,d1edf937f3d34f888a97a825cd4349ce,004ce182-0fac-4730-a3f9-70822af82d14
|
||||
1795180206.453884,warmup,201,0.010658979415893555,d760aca1b09a4234aff7b454c14917a3,18dabcdf-5de9-4714-82a2-15c3173e9eec
|
||||
1795180206.707329,warmup,201,0.009528160095214844,33ce61e42af349f9bc78467bd915b819,5ef9abc7-bf6e-4bea-a622-f4a7527aaedf
|
||||
1795180206.95679,warmup,201,0.01139521598815918,739212b4e75f419fab41b1eeec99ac92,16cdc8c3-d46c-47ea-9b71-6f75b08d15ab
|
||||
1795180207.206121,warmup,201,0.0551609992980957,7ffc640f460d4663a2828f833d49a850,a15bc994-c880-44a4-b584-4af86433b876
|
||||
1795180207.453679,warmup,201,0.0324397087097168,a7f33fedba77468e891995294eacbf6c,ff7691bd-5d00-42f0-a281-a1268e2795b6
|
||||
1795180207.712791,warmup,201,0.020277976989746094,122ada18bd024ac39b6d09e8a124ba41,351ac8aa-6f70-4695-a389-e20c5c9d6767
|
||||
|
@@ -0,0 +1,645 @@
|
||||
{
|
||||
"date": "2026-11-20T13:07:47.535044+00:00",
|
||||
"source_commit": "752a5e6c0d33c2856f7ac8e66074e4adfb9a699d",
|
||||
"scope": "full30m",
|
||||
"profile": {
|
||||
"messages_per_second": 4,
|
||||
"max_inflight": 16,
|
||||
"upload_every_seconds": 5,
|
||||
"image_pixels": [
|
||||
320,
|
||||
200
|
||||
]
|
||||
},
|
||||
"complete": false,
|
||||
"samples": [
|
||||
{
|
||||
"at": 1795180074.9828138,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 0.0,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 3.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.023879714074439243,
|
||||
"slo_good": 9.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 1694.0,
|
||||
"collector_sent_spans": 1653.0,
|
||||
"collector_received_logs": 42.0,
|
||||
"collector_sent_logs": 36.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 606171136.0
|
||||
},
|
||||
"alerts": [
|
||||
{
|
||||
"name": "ReadySlowBurn",
|
||||
"state": "pending",
|
||||
"active_at": "2026-11-20T13:07:09.758725496Z"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at": 1795180083.841643,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 1.736874,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 5.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.16976126432030372,
|
||||
"slo_good": 10.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 2093.0,
|
||||
"collector_sent_spans": 2056.0,
|
||||
"collector_received_logs": 93.0,
|
||||
"collector_sent_logs": 88.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 550273024.0
|
||||
},
|
||||
"alerts": [
|
||||
{
|
||||
"name": "ReadySlowBurn",
|
||||
"state": "pending",
|
||||
"active_at": "2026-11-20T13:07:09.758725496Z"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at": 1795180093.261457,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 3.1227,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 6.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.15639485593204194,
|
||||
"slo_good": 11.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 2492.0,
|
||||
"collector_sent_spans": 2455.0,
|
||||
"collector_received_logs": 143.0,
|
||||
"collector_sent_logs": 138.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 550490112.0
|
||||
},
|
||||
"alerts": [
|
||||
{
|
||||
"name": "ReadySlowBurn",
|
||||
"state": "pending",
|
||||
"active_at": "2026-11-20T13:07:09.758725496Z"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at": 1795180102.256551,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 1.905773,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 8.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.11976781488414731,
|
||||
"slo_good": 13.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 2698.0,
|
||||
"collector_sent_spans": 2659.0,
|
||||
"collector_received_logs": 168.0,
|
||||
"collector_sent_logs": 163.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 550600704.0
|
||||
},
|
||||
"alerts": [
|
||||
{
|
||||
"name": "ReadySlowBurn",
|
||||
"state": "firing",
|
||||
"active_at": "2026-11-20T13:07:09.758725496Z"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at": 1795180111.882427,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 2.421956,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 10.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.10091672235757894,
|
||||
"slo_good": 15.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 3098.0,
|
||||
"collector_sent_spans": 3053.0,
|
||||
"collector_received_logs": 218.0,
|
||||
"collector_sent_logs": 212.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 539901952.0
|
||||
},
|
||||
"alerts": [
|
||||
{
|
||||
"name": "ReadySlowBurn",
|
||||
"state": "firing",
|
||||
"active_at": "2026-11-20T13:07:09.758725496Z"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at": 1795180121.2686348,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 2.896534,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 12.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.11872534894646448,
|
||||
"slo_good": 17.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 3492.0,
|
||||
"collector_sent_spans": 3455.0,
|
||||
"collector_received_logs": 267.0,
|
||||
"collector_sent_logs": 262.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 540340224.0
|
||||
},
|
||||
"alerts": [
|
||||
{
|
||||
"name": "ReadySlowBurn",
|
||||
"state": "firing",
|
||||
"active_at": "2026-11-20T13:07:09.758725496Z"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at": 1795180131.386204,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 0.0,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 14.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.10737380018854843,
|
||||
"slo_good": 20.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 3883.0,
|
||||
"collector_sent_spans": 3848.0,
|
||||
"collector_received_logs": 317.0,
|
||||
"collector_sent_logs": 313.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 540286976.0
|
||||
},
|
||||
"alerts": [
|
||||
{
|
||||
"name": "ReadySlowBurn",
|
||||
"state": "firing",
|
||||
"active_at": "2026-11-20T13:07:09.758725496Z"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at": 1795180140.1158159,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 2.376214,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 16.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.14772727272727285,
|
||||
"slo_good": 21.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 4279.0,
|
||||
"collector_sent_spans": 4244.0,
|
||||
"collector_received_logs": 368.0,
|
||||
"collector_sent_logs": 363.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 540254208.0
|
||||
},
|
||||
"alerts": [
|
||||
{
|
||||
"name": "ReadySlowBurn",
|
||||
"state": "firing",
|
||||
"active_at": "2026-11-20T13:07:09.758725496Z"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at": 1795180148.943098,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 2.696411,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 18.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.13749999999999962,
|
||||
"slo_good": 23.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 4680.0,
|
||||
"collector_sent_spans": 4644.0,
|
||||
"collector_received_logs": 418.0,
|
||||
"collector_sent_logs": 413.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 532488192.0
|
||||
},
|
||||
"alerts": [
|
||||
{
|
||||
"name": "ReadySlowBurn",
|
||||
"state": "firing",
|
||||
"active_at": "2026-11-20T13:07:09.758725496Z"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at": 1795180161.9845,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 1.028664,
|
||||
"worker_busy": 1.0,
|
||||
"worker_attempts": 19.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.1250000000000001,
|
||||
"slo_good": 25.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 5027.0,
|
||||
"collector_sent_spans": 5025.0,
|
||||
"collector_received_logs": 461.0,
|
||||
"collector_sent_logs": 461.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 532381696.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
{
|
||||
"at": 1795180173.501747,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 2.296035,
|
||||
"worker_busy": 1.0,
|
||||
"worker_attempts": 21.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.2624999999999987,
|
||||
"slo_good": 27.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 5455.0,
|
||||
"collector_sent_spans": 5444.0,
|
||||
"collector_received_logs": 518.0,
|
||||
"collector_sent_logs": 516.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 533110784.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
{
|
||||
"at": 1795180182.0484211,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 1.632452,
|
||||
"worker_busy": 1.0,
|
||||
"worker_attempts": 23.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.30208333333333304,
|
||||
"slo_good": 29.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 5838.0,
|
||||
"collector_sent_spans": 5834.0,
|
||||
"collector_received_logs": 566.0,
|
||||
"collector_sent_logs": 564.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 1.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 533004288.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
{
|
||||
"at": 1795180191.050949,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 2.37788,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 26.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.26499999999999924,
|
||||
"slo_good": 31.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 6244.0,
|
||||
"collector_sent_spans": 6198.0,
|
||||
"collector_received_logs": 618.0,
|
||||
"collector_sent_logs": 612.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 533053440.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
{
|
||||
"at": 1795180202.356919,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 0.84311,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 28.0,
|
||||
"pool_used": 1.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.2662500000000005,
|
||||
"slo_good": 33.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 6632.0,
|
||||
"collector_sent_spans": 6593.0,
|
||||
"collector_received_logs": 667.0,
|
||||
"collector_sent_logs": 662.0,
|
||||
"collector_failed_spans": 0.0,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 0.0,
|
||||
"app_memory": 532992000.0
|
||||
},
|
||||
"alerts": []
|
||||
}
|
||||
],
|
||||
"phases": [
|
||||
{
|
||||
"name": "warmup",
|
||||
"start": 1795180069.957184,
|
||||
"planned_seconds": 300
|
||||
}
|
||||
],
|
||||
"elapsed_seconds": 140.295,
|
||||
"requests_csv": "slo-research-first-requests.csv",
|
||||
"request_statuses": {
|
||||
"201": 552
|
||||
},
|
||||
"uploads": [
|
||||
[
|
||||
1795180069.957095,
|
||||
"warmup",
|
||||
202,
|
||||
"a4530828-8dfc-4109-b628-c02a3798ccf1",
|
||||
"6b01f3f7a2c341fa8671f4d7a5d53fac"
|
||||
],
|
||||
[
|
||||
1795180074.9614038,
|
||||
"warmup",
|
||||
202,
|
||||
"c7630b1f-ae32-4307-85ce-1ad2b9d2ce82",
|
||||
"57b638b29f49450e8d1638504c6586c5"
|
||||
],
|
||||
[
|
||||
1795180079.971358,
|
||||
"warmup",
|
||||
202,
|
||||
"2fe8450f-20bb-43db-bbea-e440571f46b4",
|
||||
"863d1ebc8de645fca91cca1b39d7dd5d"
|
||||
],
|
||||
[
|
||||
1795180084.971487,
|
||||
"warmup",
|
||||
202,
|
||||
"3b99ce14-4b93-4446-a8cd-d19cf1032345",
|
||||
"30150967491048afbccffa77cbe1438d"
|
||||
],
|
||||
[
|
||||
1795180089.980097,
|
||||
"warmup",
|
||||
202,
|
||||
"b91d2c9f-66fb-47bb-b4da-3a65110d24ea",
|
||||
"a7740e5a4e634f0e8eb508adfe0ad3fb"
|
||||
],
|
||||
[
|
||||
1795180094.99019,
|
||||
"warmup",
|
||||
202,
|
||||
"03975eb4-f8cc-453c-a291-381d1213063d",
|
||||
"7e86598bfe5942f5910a9076c4451f13"
|
||||
],
|
||||
[
|
||||
1795180099.991049,
|
||||
"warmup",
|
||||
202,
|
||||
"67468f17-bac2-4546-8190-f5c21a12d099",
|
||||
"83aa89ae7b084fa7b44e2cee426f2b55"
|
||||
],
|
||||
[
|
||||
1795180104.9930298,
|
||||
"warmup",
|
||||
202,
|
||||
"12874504-b542-4906-9cd2-bb52d9128539",
|
||||
"736eec0cee79414fb1d20a5ae6622f7f"
|
||||
],
|
||||
[
|
||||
1795180109.997144,
|
||||
"warmup",
|
||||
202,
|
||||
"6f17982e-907c-442f-972d-4156261e0603",
|
||||
"a94be328ecf34e7da17789c6a7fa0902"
|
||||
],
|
||||
[
|
||||
1795180115.0068,
|
||||
"warmup",
|
||||
202,
|
||||
"4ddac804-25bb-4d31-b491-c34a66f834ef",
|
||||
"da4430b7065842f2a089cea6db3d8d22"
|
||||
],
|
||||
[
|
||||
1795180120.0094888,
|
||||
"warmup",
|
||||
202,
|
||||
"39b39e5c-0750-442c-8d87-14dfdbedeea4",
|
||||
"e54443c722124a3284515f71b24cb006"
|
||||
],
|
||||
[
|
||||
1795180125.0124538,
|
||||
"warmup",
|
||||
202,
|
||||
"ae29301a-cd3c-49a9-88c7-f96d42eff4d3",
|
||||
"d64a7beb706844038d898951bb059c2f"
|
||||
],
|
||||
[
|
||||
1795180130.01914,
|
||||
"warmup",
|
||||
202,
|
||||
"a3d56ad4-c89d-4c09-880d-effd96fd2fc7",
|
||||
"982067e9b4bf4dfd9c72ac87c8054086"
|
||||
],
|
||||
[
|
||||
1795180135.029181,
|
||||
"warmup",
|
||||
202,
|
||||
"ebe4358b-e000-4341-bcda-5ccb5f4016b9",
|
||||
"3625b14d985242efbdd86fde7db2d2cc"
|
||||
],
|
||||
[
|
||||
1795180140.030897,
|
||||
"warmup",
|
||||
202,
|
||||
"39507a43-8917-4c79-b3cf-31931e82813f",
|
||||
"e93edc815c2540a280185aa9a5e57ec0"
|
||||
],
|
||||
[
|
||||
1795180145.038631,
|
||||
"warmup",
|
||||
202,
|
||||
"78c052c3-1321-429b-b69e-994fbbc04738",
|
||||
"d9f9d2bdf0064f97bed896362b878e50"
|
||||
],
|
||||
[
|
||||
1795180150.043772,
|
||||
"warmup",
|
||||
202,
|
||||
"df3768d5-8727-4083-8e0b-280c278b9a5e",
|
||||
"cbe3f59ea86e4917bcdd5dffae7fa391"
|
||||
],
|
||||
[
|
||||
1795180155.116708,
|
||||
"warmup",
|
||||
202,
|
||||
"38e19499-2f53-429f-9f39-696f8ddf9d78",
|
||||
"d5b874084df34ceda4a536b8704e3577"
|
||||
],
|
||||
[
|
||||
1795180160.1169658,
|
||||
"warmup",
|
||||
202,
|
||||
"fd1ab7f5-fefe-463b-97f7-2b909cd314d2",
|
||||
"0a6d63e813c3456c9e26f14863b13067"
|
||||
],
|
||||
[
|
||||
1795180165.117381,
|
||||
"warmup",
|
||||
202,
|
||||
"aef650a9-1573-4d0d-9a9c-8562e9e3b544",
|
||||
"5749949cae4f47e59d8f5b4800937e05"
|
||||
],
|
||||
[
|
||||
1795180170.123688,
|
||||
"warmup",
|
||||
202,
|
||||
"e52a9186-83c3-40bd-9f0c-05dc56a653d8",
|
||||
"1922cb0439244afe9050bca802cae6e1"
|
||||
],
|
||||
[
|
||||
1795180175.127093,
|
||||
"warmup",
|
||||
202,
|
||||
"ca907578-1425-4536-91cf-8bf7a39645ba",
|
||||
"16041423c09e4c6cab733a7c939543ce"
|
||||
],
|
||||
[
|
||||
1795180180.128174,
|
||||
"warmup",
|
||||
202,
|
||||
"ee979ea1-bf9e-4ea0-9e99-9af941030d60",
|
||||
"501f6d91de9e4c6bab4dc0c9b574745e"
|
||||
],
|
||||
[
|
||||
1795180185.137634,
|
||||
"warmup",
|
||||
202,
|
||||
"bd9a05f7-e2a5-4113-9d2c-805ee37be00a",
|
||||
"b8fb5b6793ac457ba6ef67830fb3a3eb"
|
||||
],
|
||||
[
|
||||
1795180190.142824,
|
||||
"warmup",
|
||||
202,
|
||||
"a7d987c0-f74c-467c-8e77-129e1741d733",
|
||||
"47ce6dee6f8e45a9b6107b59170dfa0e"
|
||||
],
|
||||
[
|
||||
1795180195.147142,
|
||||
"warmup",
|
||||
202,
|
||||
"a8b31d86-325a-4398-a14e-d406b6644355",
|
||||
"16760f23381d4a788f486ff1600e47e3"
|
||||
],
|
||||
[
|
||||
1795180200.147872,
|
||||
"warmup",
|
||||
202,
|
||||
"7eea348e-417f-4cf3-8097-73211b00d1ef",
|
||||
"af7fe6002e514612a625962d360970d9"
|
||||
],
|
||||
[
|
||||
1795180205.1478279,
|
||||
"warmup",
|
||||
202,
|
||||
"7de6cd57-c67f-4076-9972-7f3a539a5f79",
|
||||
"46ecb113b6314b928232eb5bfece1cad"
|
||||
]
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
Traceback (most recent call last):
|
||||
File "<frozen runpy>", line 203, in _run_module_as_main
|
||||
File "<frozen runpy>", line 88, in _run_code
|
||||
File "<repository>/scripts/research_slo.py", line 631, in <module>
|
||||
main()
|
||||
~~~~^^
|
||||
File "<repository>/scripts/research_slo.py", line 479, in main
|
||||
max(s["signals"]["collector_memory"] for s in points)
|
||||
< report["container_limits"]["otel"]["mem_limit"]
|
||||
TypeError: '<' not supported between instances of 'float' and 'str'
|
||||
@@ -0,0 +1,223 @@
|
||||
timestamp,phase,status,seconds,trace_id,message_id
|
||||
1795180306.423019,warmup,201,0.055117130279541016,76b17926effb4139b48ade994fd1b7de,45edeac4-7cee-4570-b117-9b26c569b543
|
||||
1795180306.674056,warmup,201,0.11263298988342285,c6b3b19c02944693b96797163527546f,f9e4dcbc-dae1-4e5c-b8a2-74edf26846cb
|
||||
1795180306.925218,warmup,201,0.024322986602783203,3eb10d8793dd4669ac70599f8be9eb90,0a3a27a5-1178-4faf-a664-4702712e783e
|
||||
1795180307.181909,warmup,201,0.06856274604797363,3a0245c1ee554cbeb5ad64de9ba6e75c,102398b3-cfcb-4686-b2d8-10994305cc1d
|
||||
1795180307.4298592,warmup,201,0.050598859786987305,f2bd58199b5245a685b5fda701ecff3d,48d12936-0916-4eaa-aaf9-05b2c364c87f
|
||||
1795180307.6729841,warmup,201,0.016305923461914062,ff9fb6fe48aa4a87bcdc079e014d31b6,ef71bafd-8db5-4b6c-9c67-d2f630135491
|
||||
1795180307.9231732,warmup,201,0.0525507926940918,14318f14c2c8420ab8d4d3ae3e0bcf40,ace23a9f-f216-4282-b3ae-2a7985d8f37d
|
||||
1795180308.180491,warmup,201,0.034638166427612305,10944fea4ef94741ac5dde89779d2783,9f34c3fe-40fb-4d81-8140-5722ad929244
|
||||
1795180308.422879,warmup,201,0.0330810546875,584c358735fc4175b803475f58b3670e,83dae3dc-900a-48e8-8bdd-2dca4a879315
|
||||
1795180308.672776,warmup,201,0.02603316307067871,92fa42f3977b423ea6405fe6c84108c5,c58e9f22-94dc-4b9b-aa6f-e58fa4bd4ce5
|
||||
1795180308.927267,warmup,201,0.021506071090698242,16b96bd7db7648ad81c67d33af6da58e,5b0e326e-ad2b-4a9a-ba40-89e16ac238cf
|
||||
1795180309.178731,warmup,201,0.04657888412475586,b894046a5e24431db9777227f00e5e5e,d76724d8-0b6f-4014-9331-5104e76286ce
|
||||
1795180309.426372,warmup,201,0.01767587661743164,9c914e9234234f6b9ec5aebd0700b08d,e19cfe8c-439f-4094-b088-78a5fa598a49
|
||||
1795180309.6781561,warmup,201,0.034906864166259766,aff7ab332dd44350a77bbe170ea2254c,bd24569d-e245-47a4-8713-61982c26b8e6
|
||||
1795180309.923274,warmup,201,0.06194877624511719,bafc8d9e97124b0788c1d0ebd258e03b,bcec2bbb-2014-4d77-a740-35c279e875ab
|
||||
1795180310.1744561,warmup,201,0.27138471603393555,792f1a0bbbcb40439b4908a661f2453b,cdd7e2ce-bae9-4430-9f77-21dba2d63759
|
||||
1795180310.4293928,warmup,201,0.03658699989318848,62b9f45e808e4001b8d94f32a951d09a,d4a2ef1b-2bb1-4d75-9c31-6b029bdb1cfb
|
||||
1795180310.682456,warmup,201,0.027057886123657227,89af845a6be644b6a3052232976be957,4c2264e0-0b23-4884-bd2d-72aa066894c9
|
||||
1795180310.931253,warmup,201,0.017241239547729492,e5c8386c382c4b3e9ead235dc2581b04,7f90ab2b-c35b-4ce1-a813-7320024cd8d1
|
||||
1795180311.177042,warmup,201,0.0361180305480957,ecd9c03f4e884f4b82acbea5cb6612d2,faa61c28-259f-40e4-aec8-ecb33dfbdac3
|
||||
1795180311.426838,warmup,201,0.0326380729675293,f686d484d73448a3b95729eac184bf66,7c150bc6-edbd-4e39-befa-b60f3d905972
|
||||
1795180311.6764312,warmup,201,0.15946173667907715,b9dde373785c4e048354ed3ce67d3300,db20ab6a-f67d-495c-9fc8-d8df2b123fe0
|
||||
1795180311.929388,warmup,201,0.044622182846069336,ba4dfdcff45b4c53a7a07904fa61f94a,f6be3558-4eee-459f-9014-3838e8393b1c
|
||||
1795180312.1749911,warmup,201,0.07517695426940918,957ff476cdbd41b290da286dfcc60b16,fe032f0d-e2c9-4550-afdc-a1d099385366
|
||||
1795180312.4307652,warmup,201,0.020746946334838867,dbfdb96f8ac44c58b1160e0fa42a6984,c4538789-36b3-43bc-9d95-b56d017ea9c5
|
||||
1795180312.67829,warmup,201,0.06154799461364746,a905d5b3feaa437e91ad1299ae1ed7c0,1d9a4f48-5605-4d6d-8021-9cc92ab58f45
|
||||
1795180312.928323,warmup,201,0.011504888534545898,972030cfa8694a50a7f8abbafb7a8f87,dadee304-20e6-42df-9108-46b53634cb8e
|
||||
1795180313.181609,warmup,201,0.014528036117553711,5a58df298b354208bac9300c340649d6,d9e7e3ce-2ea0-4948-804d-c28f16f047b7
|
||||
1795180313.428823,warmup,201,0.021541833877563477,7e4a6d7bb2b3441bbaa56c17f376bd1c,942cdccf-9cbb-473b-aca0-f6018b42aad8
|
||||
1795180313.682583,warmup,201,0.011409759521484375,2ab48b9886874f7ab4b224e5e0fb47df,0ab859a3-61dd-42d6-b32d-f2dd92129bc1
|
||||
1795180313.926015,warmup,201,0.09591507911682129,25a7bbaad50445f99471f406b5afb9a4,538117fb-17dc-4dbd-8708-82a431a28102
|
||||
1795180314.174363,warmup,201,0.022596120834350586,a0366368908c441e9a0d41543390ef78,37e7f867-90be-4c6d-97b4-250ce02b1328
|
||||
1795180314.4236832,warmup,201,0.01119089126586914,1c295becc4334f6dbfd8448b1c7106a4,948c8e31-2d92-4a2e-94b0-67a47144368e
|
||||
1795180314.680267,warmup,201,0.01999187469482422,e512543e7f7d44ce960de76da5b17cc3,0614e562-4376-468e-b84e-76e904a4218c
|
||||
1795180314.929228,warmup,201,0.019069910049438477,b7a3683322474a8a8f066d0664b9d431,175c9e9d-eff8-4e12-8ec2-27c3450f0f0f
|
||||
1795180315.180566,warmup,201,0.015458106994628906,cf5a580d5c08459996d3d8231b807ad4,127952d0-8a07-48a8-b807-26dc230ae1a9
|
||||
1795180315.425687,warmup,201,0.018053770065307617,22942e117b2e4d958b71083c877ef01b,aae0dbb8-a3a9-457a-a0d0-68bae6960d7f
|
||||
1795180315.675526,warmup,201,0.026155948638916016,721fe3c04e7b480394d2d6ddaa8e1a6a,53dd64c2-b25f-4ae9-a170-a840e3ce21bc
|
||||
1795180315.924555,warmup,201,0.011884927749633789,7f06f138058e4a1484b29ade9a22eba5,27a4defe-4e15-4ecd-bc31-ed2c75d18d83
|
||||
1795180316.1811168,warmup,201,0.032523155212402344,0e0aae0cd32a42d9b962da5117e36666,b0d5c7c9-60e5-4d86-b4f2-703fa9bfdcc6
|
||||
1795180316.4279091,warmup,201,0.018095970153808594,a6f9a0c0acd244b69d902cb79725d373,573c9fbf-70c9-4fa1-a992-727904d1e568
|
||||
1795180316.673577,warmup,201,0.009534835815429688,94cefc2182544e3981144d87d4d4fcee,47ffbbd9-a1ad-40c6-8466-884c598607ca
|
||||
1795180316.926636,warmup,201,0.014694929122924805,5bbef0350fb64b6bac71b79770ea13f0,52d9bb5b-d73d-4952-8c59-446b262912e1
|
||||
1795180317.182466,warmup,201,0.015308141708374023,74e404a445284782b86afc86fe4644b5,25f85804-c3f2-4585-a2ea-cdee0c7fc7d2
|
||||
1795180317.428221,warmup,201,0.010089874267578125,3495bd89b7ad4f349515b64c09374f81,6b7d028b-30e7-49db-ae12-4abca73e816a
|
||||
1795180317.678735,warmup,201,0.014368057250976562,192e67e8fd5244589801a03e0771ec4a,13aade2d-bba3-44ed-83f2-179be7eb03b0
|
||||
1795180317.9327989,warmup,201,0.022715330123901367,b0f30a0f95c84b15823b2bac102c8d0f,df64a7a8-fa8c-4743-9c35-a2711710139d
|
||||
1795180318.1822689,warmup,201,0.011677265167236328,d7ea7d3721e645278cff25c11e34d4de,ebc15bda-434d-479e-8951-0e4bb696082a
|
||||
1795180318.4249558,warmup,201,0.02953815460205078,7989769fa68546c89900ef7023cdfe52,a66b2086-50c8-4c87-a3b8-63f4d8aeca8d
|
||||
1795180318.678638,warmup,201,0.013857126235961914,d8cd43459576487dad26c79c97c2292a,05a4b084-9a87-47e4-873d-349fdd3a4643
|
||||
1795180318.9239738,warmup,201,0.014153242111206055,00d13065595845af8b4b55052390c819,cfb927dc-00a9-45dd-9734-aaa4f1f0cb22
|
||||
1795180319.1802251,warmup,201,0.0406038761138916,e10654f795ae4419a4b812aadb96970e,f3accec8-f106-413c-ac14-a63a943aa726
|
||||
1795180319.429311,warmup,201,0.01441502571105957,19b74567fbe747fdb24d7d06efa6d1eb,2ceb1a29-52d1-42a1-9565-78acfd5e9fc8
|
||||
1795180319.6822212,warmup,201,0.008408784866333008,93cb6031e83747bc920fc2fb9610661f,5b60fc86-3a0a-4df9-96a9-b2631210d432
|
||||
1795180319.923309,warmup,201,0.03721809387207031,7ef202b3302c499ca0910961fd7a3433,aed217bd-aaff-44d5-b5bc-b704d0e37234
|
||||
1795180320.1749089,warmup,201,0.060726165771484375,38326ddf0825495a8902d615734748bc,444bac02-07bf-4e3f-8e88-f7abc4b3d347
|
||||
1795180320.4323468,warmup,201,0.027462244033813477,a334076530fd46cb9c146528fc43c6a8,64d62ba8-4512-41ef-bcda-2e5e33e996ec
|
||||
1795180320.6773481,warmup,201,0.03374075889587402,0b2a7694ca2b45ba912289c2deedce69,0f8b0cce-bf64-48ff-a038-7a2d3a32c280
|
||||
1795180320.924204,warmup,201,0.013178825378417969,3454ac5fb3304fe7b53bc8445b0d05d1,4bb1b2fb-e11e-4a28-9330-040f6cb8d25c
|
||||
1795180321.174137,warmup,201,0.010026931762695312,dbf03db4b1d8440587d5a1f60d5ea14f,09e09bf5-02a6-451f-82a8-769e6aa0facd
|
||||
1795180321.428226,warmup,201,0.011801958084106445,ca76f22c791748d48455ca5f7381661d,0a7de369-03a2-44eb-8c15-b1c53e13fbc8
|
||||
1795180321.679889,warmup,201,0.010977029800415039,dd6f8475d4ed4376868c4405935692fd,35fee93a-8bd8-4824-9822-6ca26266e5b0
|
||||
1795180321.9291892,warmup,201,0.011880874633789062,6096d515e8f1422f81419c5fb1a124ef,bfb8f12f-9be1-4089-8b93-d2eef7625675
|
||||
1795180322.180209,warmup,201,0.0462651252746582,9b12c16b30024558b2f47cecfddf1f82,d7ac3615-aafa-4ee6-86a3-7cd9497fb093
|
||||
1795180322.422112,warmup,201,0.05217790603637695,2c18bdf8d4ca47ec97d0279c968e2ea6,c308ad03-e992-4ef0-8ca0-8faf408f1650
|
||||
1795180322.68032,warmup,201,0.01891303062438965,a9088dcd6c5b4fb2993d9b8131868f04,8fab0bdc-1125-4e0f-9645-e040744f5730
|
||||
1795180322.924831,warmup,201,0.011192083358764648,363cada73f8e40b3863498d521f84103,61491b7d-7e26-4ad3-af54-759104015bfc
|
||||
1795180323.182313,warmup,201,0.06261301040649414,b868e760d7434f16b95e07fc3c036e03,494387fb-8db0-494c-b720-a63d0d52eeb5
|
||||
1795180323.432374,warmup,201,0.01579904556274414,e8e2992ee027446a99d04e7ed54cc5fb,1d69df6e-a939-44bd-9cc4-0e1e87ccaab4
|
||||
1795180323.682173,warmup,201,0.02521991729736328,11093f1c7d764d229815e1b41b456396,da181d3e-a345-4a02-96c7-463d144f0a96
|
||||
1795180323.925667,warmup,201,0.016142845153808594,6d3e0801fec94b5da2834f7a736bf56f,16f9b4e6-f3fc-4c0d-8432-2e81ec0bcc13
|
||||
1795180324.174363,warmup,201,0.0512690544128418,1d99895f9e434de3843e0248703eb1e4,89d42939-6925-4cd8-88ff-288e1dc85428
|
||||
1795180324.4281669,warmup,201,0.021039247512817383,c8b92c7d5d56452faa59c4d2ba09197a,57b8846b-51f2-4d28-9a88-14eb08a73817
|
||||
1795180324.674187,warmup,201,0.021563053131103516,78665560fdef454788a688ff19ba0ee4,e3945170-8574-4883-a3d6-73ed17abdc56
|
||||
1795180324.926713,warmup,201,0.09740400314331055,eefc3558c2a54ec19a5bdcbffd5872ff,4b3ff7a6-26bc-4a4f-b6f5-5b20e27e8ba9
|
||||
1795180325.179863,warmup,201,0.0670158863067627,10d8be6f1b094b4a860825c5cdcb66bf,1abc8596-c125-448f-b6bd-f56e5db13cb8
|
||||
1795180325.430054,warmup,201,0.07341194152832031,baf1cf9b30174a0eb160a4d564f15813,cf8c046c-7e9e-4448-90dd-dd294ebad9ac
|
||||
1795180325.6817129,warmup,201,0.009120941162109375,6a862f33186c4cebbb25ff98178f2bfa,95527733-2c5a-498d-9f49-f8d301d4d1c3
|
||||
1795180325.926674,warmup,201,0.012025117874145508,cd82070cc11f4ef29b911375577865c3,486bf416-ff9b-4597-bbbd-dc75a7c30490
|
||||
1795180326.182048,warmup,201,0.013367891311645508,5eebee0de5624d7985d7565367d7c6b4,5945a92a-1095-49a5-9d13-29097b3f622c
|
||||
1795180326.428158,warmup,201,0.013689041137695312,f9d615eb908743ecb21146371ff6f4c2,d52b0441-fbeb-43e8-a95f-08c455160db5
|
||||
1795180326.672092,warmup,201,0.010310173034667969,1b7f0b149b4c4b1dbd9904657ebf0ae4,af5aa204-1c19-48de-bb9a-c78fb8290f9e
|
||||
1795180326.9302368,warmup,201,0.015584230422973633,c3b77b73036944bcb20956e0ba2f6a90,ebc74447-e1ea-4c86-8e0d-697d5195b2db
|
||||
1795180327.173146,warmup,201,0.01581406593322754,90ba92757b584b6fbe0358a4df665ab0,6c928cc6-7a0f-4b4a-901e-edd2b8a5c83f
|
||||
1795180327.427222,warmup,201,0.012076139450073242,d1daf4f998454f4ea6d0297bda8f6e04,607a7eed-38ac-4965-9478-6895a83e8fdf
|
||||
1795180327.672278,warmup,201,0.010951995849609375,9ec736838ed949fb83a63200454fea36,0d74bd92-7493-43c9-a354-0c65d3b97e97
|
||||
1795180327.9319308,warmup,201,0.011237144470214844,fb5eaf9967284cb397cbceae4661dddd,9e8c6eb6-ce68-43c6-9e92-8de1057ce3b1
|
||||
1795180328.175093,warmup,201,0.01266622543334961,35270abcaff847faa9429abd5c4de7e6,aae33024-4f87-4108-8cd6-63ff8766e9a7
|
||||
1795180328.423997,warmup,201,0.13264989852905273,b9780a4af98a41baaccbb8106999db5d,80ae627d-9a2d-4e82-b729-4f707b1f21c7
|
||||
1795180328.6788118,warmup,201,0.045153141021728516,d5ed1030e5bf401caa1f2a5adc0873cb,4dfd91fd-2d14-4ad9-9114-de44974aac8b
|
||||
1795180328.922585,warmup,201,0.014145135879516602,9a3d7c283964490089d1c6d261185589,62e1ba19-45d2-41a8-b841-a043110f7704
|
||||
1795180329.178997,warmup,201,0.022284984588623047,8951f43b75ec4005a7608f31c70c26a6,beb8b635-f948-4dbd-8ff5-1265fde89b09
|
||||
1795180329.424947,warmup,201,0.04422593116760254,9240a763e0e94a62b52ebfc4ad708cc4,0c6428e3-de7b-4c53-b446-a3665b141b7f
|
||||
1795180329.680489,warmup,201,0.03175187110900879,7f8fc03fa51e4dd2a37339ac606ffecb,fd540e12-8e6f-4374-ab1e-d52b34d78e4e
|
||||
1795180329.9236531,warmup,201,0.013557672500610352,98c5a9f09f044dcbbff494387bafbd4c,93589d91-8865-4cf8-8bad-58564a889d89
|
||||
1795180330.1782389,warmup,201,0.02124500274658203,1e695d9d2c2f48b88dfd76f8bf8cf490,5d160787-23f3-4e62-94a3-cdf9b282a871
|
||||
1795180330.423929,warmup,201,0.03000617027282715,9d4e91db41fc44d2891be75670dfabc9,44d47b59-ba1f-4c2a-9d33-9791ea4db19c
|
||||
1795180330.6799638,warmup,201,0.21182918548583984,6cbf676363d54de9bbf98522a6457cc8,63f7279b-586e-4fa4-9dcd-702c937f806a
|
||||
1795180330.9291131,warmup,201,0.020792722702026367,7fd6e1c7fbc644bfab8092c898d748e7,672e9274-7ab2-4667-b35e-af3bd8b2b036
|
||||
1795180331.179322,warmup,201,0.046911001205444336,021915e1b41b4f34ad85f3d2e6a4da0f,dce01763-b702-4ce9-9ee3-921e4b3c9886
|
||||
1795180331.4241679,warmup,201,0.05442523956298828,99a3588c6c91443bbf6bbc2cdcac7d2a,29cc6791-e6fe-4f9e-992f-4479215fc266
|
||||
1795180331.678278,warmup,201,0.017962217330932617,99d25a5eb2064395927eb09e8aa222cf,8a16a0e9-0219-44e8-a920-653874d0b8d9
|
||||
1795180331.92304,warmup,201,0.018758058547973633,cffa0a7dd9bc4236a6038010bf40cccc,ea7f502a-39f9-418e-9fea-7906409ed598
|
||||
1795180332.176664,warmup,201,0.012223958969116211,559e362406b84895a79a3cbaf54d49c2,713a7dce-5626-404d-9662-121e875c8528
|
||||
1795180332.421847,warmup,201,0.015980958938598633,20d2755def3649c98662f8e7142e2681,415f4a48-ebdf-4511-abe1-deabbc7d73a9
|
||||
1795180332.67451,warmup,201,0.029402971267700195,ae57e3232d8240baa8a8cce8ab650325,54a8e098-a1f5-4b6a-9e43-0b056bb3e2e6
|
||||
1795180332.930043,warmup,201,0.011388063430786133,cee81cf842a544e2b1700c6d5e5f4cbd,731cf770-d0eb-4aec-8aed-f5949290a301
|
||||
1795180333.172213,warmup,201,0.01586604118347168,bf8d441585fd49b788c92dbfd279d25c,47cafb93-d453-47e7-85d2-ead4ee14c814
|
||||
1795180333.423415,warmup,201,0.014913082122802734,aee265c80d76491ca511c2dd67e5bef7,4e051465-0aed-418b-874d-2fa35311e668
|
||||
1795180333.67177,warmup,201,0.011767864227294922,b5b423c76c1a43fb8883ea4801d396dc,42fdaa69-7e20-43a3-8b59-7a638f9621b6
|
||||
1795180333.928659,warmup,201,0.013628959655761719,5c0752fad40d4761bfde52ee54822e58,afc00df1-4d20-412c-872c-d0d320cf3d37
|
||||
1795180334.175783,warmup,201,0.07439613342285156,478b50c50a3f4fe59b7aa4962d110637,ad0585cc-a72f-49dd-8545-bb0935f21245
|
||||
1795180334.428273,warmup,201,0.013356924057006836,b0ffff00bf0a4f02a7e8de3e83944f0d,1c24e631-7a77-498e-a625-8538d60f62e5
|
||||
1795180334.672097,warmup,201,0.013913869857788086,be440f8a05d3492a9a4816f7aa2c02a0,04df633a-94bf-4348-b397-ac7b59937040
|
||||
1795180334.929533,warmup,201,0.6379430294036865,d8e4bcffdbd24e47ad264517c65f0dfa,6963c7af-7de7-4700-89b3-4c4bd02de125
|
||||
1795180335.4310532,warmup,201,0.21014761924743652,e8a7cc4c3dba45daba6cabedfdbaabf0,87999bae-e07d-42e8-86c0-4a457dfabf0b
|
||||
1795180335.179592,warmup,201,0.46358299255371094,198efae63dd14b83a1f67c8256b36fb2,2d5926f4-0437-4983-80f7-93f33a3c2dda
|
||||
1795180335.674002,warmup,201,0.05605912208557129,c30042701c1f4f1eaef7bdc88b3920f7,f8a28688-8d60-4dcb-8d19-754ec9d586d6
|
||||
1795180335.929294,warmup,201,0.00999593734741211,38366b0cc61c42fda055656fc8b9405c,e8f7ea7c-71f2-4463-bdff-d16a901bb8f5
|
||||
1795180336.1780539,warmup,201,0.012089014053344727,b1ef2220cce54480a9236bed82c24ff5,2d1c4a5c-7368-42ee-9fd1-f84cf3207222
|
||||
1795180336.424557,warmup,201,0.01134800910949707,0561752001b1407a9da166b663bee655,763b99c1-ded3-4502-82f3-508af1b4b232
|
||||
1795180336.681586,warmup,201,0.012669086456298828,b2507ead5d2f41548c5513101f01c681,a70bf5b4-cacd-408c-83b8-f73d8b9c6663
|
||||
1795180336.92322,warmup,201,0.025698184967041016,42c5d403c13243e48ab3497efd64efe1,d1f46a4a-7792-4438-878e-b172420013ae
|
||||
1795180337.175337,warmup,201,0.025685787200927734,732a1d48efac4a8182e803fae21b14b0,f8d80429-8488-4674-9a2c-164ff01ec638
|
||||
1795180337.4312031,warmup,201,0.012318849563598633,551a0437a6684e04858e3e9e18e2e27a,449a474d-4d08-44bb-8773-023f2ffe5ebc
|
||||
1795180337.676717,warmup,201,0.011137008666992188,58204786011c47978fc1a737c82d2ee0,740dc698-cbfc-4352-a7d4-4eb229b12cd8
|
||||
1795180337.928061,warmup,201,0.010760784149169922,d5beaf3bbcd441fc93b3614c408f70c7,7477d17c-61fc-42a6-8192-e361298e1050
|
||||
1795180338.1814332,warmup,201,0.008683919906616211,8da12b3fd0d84ca2b7e9b8b8399e6689,55629cb3-8f72-47ce-8870-02d6f8fb2e11
|
||||
1795180338.4314299,warmup,201,0.012953042984008789,dc03191b57b740c7ba16770974d28edc,8802a764-3bb3-4474-a456-aa16351ad526
|
||||
1795180338.674549,warmup,201,0.011281728744506836,501a10a665cd41ff9ca6f1bdf261b436,4af67448-4e3b-4100-a970-6aa2c28a60b4
|
||||
1795180338.921731,warmup,201,0.035794973373413086,9458f35691df40d6817b1cbc7884da09,a9b1b50c-3ecb-4e40-a233-4a84efd565f4
|
||||
1795180339.176345,warmup,201,0.01884603500366211,2424406ec5d043259e8ac310ab765f6e,0bdea318-c103-4d5e-b476-274c4feaef9b
|
||||
1795180339.428579,warmup,201,0.011592864990234375,50aaf29486064b24b9bc50e652f46cb9,edc06291-9fb4-42bb-a501-d6c1bf6f09a0
|
||||
1795180339.680001,warmup,201,0.009851932525634766,c6dce8e46aac45b8aefdefee3ef007f1,5a335b34-fef6-44bf-bda2-6a1b5482d587
|
||||
1795180339.931456,warmup,201,0.010718822479248047,8b8b563bf97a4a9e9eb24d1e3f1585f2,ff857b95-1b74-4f86-b405-f7314690b2d8
|
||||
1795180340.1814861,warmup,201,0.011487007141113281,7187e721be584781a50e5c622995bec9,f0a80079-1e90-4407-a747-a00800379415
|
||||
1795180340.4302,warmup,201,0.019636869430541992,b91a363608904d27bd4d5f4cd801b2f3,b658efea-ec64-41d6-b268-73810fc09974
|
||||
1795180340.671793,warmup,201,0.015497922897338867,cb3272c4ea774827b839828948ca594a,be030a11-9ee2-4ee3-9f20-e0e124351756
|
||||
1795180340.9280891,warmup,201,0.010939836502075195,c3f948a801ce48e1be201fb0a71efbbc,9190bf06-a491-489d-83e3-c12a518ad08c
|
||||
1795180341.1780741,warmup,201,0.013393878936767578,57c094a056fb4901bcdbd9f98cb84035,64f9cb7e-3bec-42bd-976f-22f463257144
|
||||
1795180341.423362,warmup,201,0.01970696449279785,f45ebf0ce50c4728a612832e45620dfa,82024df4-278e-4726-84e1-ed4f4c5ab451
|
||||
1795180341.6748412,warmup,201,0.008790969848632812,ae5a9f1e20fa4882b0190f619832f229,123057d3-1867-4be0-97f9-81638814ae9e
|
||||
1795180341.92869,warmup,201,0.08092522621154785,cf7231989bff434eb919c723499df91b,f68c3ae0-ca25-4b9d-ad9d-d8ab866a052d
|
||||
1795180342.175219,warmup,201,0.011922836303710938,095db27bed494b6780624dcd0ab805cb,847c10dc-bcc6-42ae-b2a6-c1cf38aad0ff
|
||||
1795180342.424076,warmup,201,0.011336803436279297,9db82bcc7a2349e6a028a579b59f3731,2ac0cbf5-c8a5-4dfc-9f00-9e53cfe74ba1
|
||||
1795180342.671635,warmup,201,0.012029886245727539,4b4531114e8842118cb57552bfe8580b,c48917dd-b068-4a03-b8f8-c8ba300ea444
|
||||
1795180342.930646,warmup,201,0.03807401657104492,77c386ecd5ad42138dbf01ed9a1a08e3,530fe2ca-019f-4498-b28a-1b46003b42fd
|
||||
1795180343.173782,warmup,201,0.02783679962158203,9fa8bd612584480b8b8d54f9d97f84c6,71a2ea1a-a408-4b76-813c-6cc112518100
|
||||
1795180343.431467,warmup,201,0.015299081802368164,ea11b713e9a34a16aae02e94ad918c04,8a705499-b818-4059-bf44-c742cc33385d
|
||||
1795180343.681278,warmup,201,0.014876127243041992,06e399d5a290466fab9e5d633b60dd0a,4dcb497e-4b72-4523-9898-a2700a14d355
|
||||
1795180343.923985,warmup,201,0.01096200942993164,cc43e56b4a2b4c19b2b6101a50c2bc94,48490016-195a-4a81-8f04-ba808859df9c
|
||||
1795180344.171476,warmup,201,0.03735709190368652,e39608915fcb42debbe456471d2da56d,6fbc4730-e650-470b-bfc9-ffc4617929af
|
||||
1795180344.4280179,warmup,201,0.020684003829956055,cef374c710c2463281482742a9d4f9d5,562998c0-4619-476d-9d87-0205e598889a
|
||||
1795180344.6780381,warmup,201,0.06757974624633789,3699dadfb0c146679e5b1d18cf0ab47d,0ffc8be8-a8e3-42c4-8315-c078810a2ff7
|
||||
1795180344.930443,warmup,201,0.01471090316772461,97b7b81f51994c4e9e513b189a6ffd37,a7a27dc8-522e-4567-961c-0db57fdea6db
|
||||
1795180345.173086,warmup,201,0.015903949737548828,68b24d7b4e6e4d16ae9506091f733fb3,c63ccb84-a500-423f-861a-6de057114f5b
|
||||
1795180345.42819,warmup,201,0.01735401153564453,31f7ff52721e4e61aa77a46238e37f5d,0a9ec936-7f43-45fb-8b0f-b66d7f4a7476
|
||||
1795180345.676356,warmup,201,0.02088189125061035,d1f916e4f1934981a8fd1c3884230435,5ebfe76f-62a3-4ed8-ab48-710d781cf530
|
||||
1795180345.92798,warmup,201,0.012233972549438477,dda6e0e723374b63a94ec455b1842d6e,aa9b085d-60fc-474b-a417-e6da81114221
|
||||
1795180346.174879,warmup,201,0.020541906356811523,00d03e3aab9e43efb45b1b99a77dd1e5,62f165a2-06ff-4919-862c-463502cd33eb
|
||||
1795180346.4241002,warmup,201,0.01257777214050293,a9631cd4218448598816db46e05ac210,1a359611-1102-4cd0-a8a8-19f739c2d5f6
|
||||
1795180346.6805272,warmup,201,0.031142711639404297,d6326c50cf6a4db4a14a959f047aec67,229641cd-c43f-42bb-bceb-a636fcc2e025
|
||||
1795180346.923172,warmup,201,0.013450145721435547,fc9078ab07b6497da577535c92e91ceb,85107c40-5384-48dc-bf32-18b48892a006
|
||||
1795180347.1747231,warmup,201,0.05333566665649414,07f0bbeaa6b44c61a7c351107b41b237,4ae310e9-fa62-43c6-b1d5-5330e738b353
|
||||
1795180347.427953,warmup,201,0.014930009841918945,202c8996cce944b9a22892a908097a34,53140b2b-a88d-4fa6-899a-a17775f5dfaf
|
||||
1795180347.680614,warmup,201,0.023784160614013672,907d1193824c42e5b30c45ba2634e043,ec60f205-beb2-4c00-9e47-1dd4c7a421fb
|
||||
1795180347.926591,warmup,201,0.010828018188476562,6adcea95ee4c4ed1a2ece269f55dbedf,156e019b-22eb-4dd0-bd16-10af4e5d41a4
|
||||
1795180348.173774,warmup,201,0.03443479537963867,f67d7dc601194d20af0034748e2c835d,a2e2b4c1-d602-4451-990c-3480b5861430
|
||||
1795180348.424813,warmup,201,0.025806903839111328,32f93effa4e745e6ba519405061bf598,1d99fe38-930b-4373-b8b1-c7a50e5883d4
|
||||
1795180348.675287,warmup,201,0.019309043884277344,ae8cd26f20be4a35b3f63d7b46a8b960,d31bebc4-a124-4ae3-abc2-af41e2d5eae6
|
||||
1795180348.923858,warmup,201,0.010105133056640625,bc0cc3806d7f4a919e5f47cda40b0cb4,19928ca1-9013-4dc2-b4d6-72ac6a1953e8
|
||||
1795180349.177574,warmup,201,0.018340110778808594,786e7738d9c748d9809bd84ff34b95c4,4642a90b-25bd-491c-bf74-e6bb9890d5d6
|
||||
1795180349.429085,warmup,201,0.04597902297973633,b1166b143286416284b919b7ee8beb1d,415730d0-e983-454a-9099-29502d67c8e4
|
||||
1795180349.678473,warmup,201,0.12787294387817383,73a76ddd1f284f46af12cb353cc71ada,7197c218-1cc3-46a9-b2c0-77200ea16b4b
|
||||
1795180349.921549,warmup,201,0.009884834289550781,d5c670be06a745ffbb46333a26e750f6,63e5afa5-b537-4bef-b6f9-02467ead6dad
|
||||
1795180350.180325,warmup,201,0.014764785766601562,163c2435e5b84ac3a6d9925094a8896b,136194ea-2088-486c-87d4-d0b7644f17f5
|
||||
1795180350.424948,warmup,201,0.015136957168579102,c06a3f595cf54357a11aa8e6e99ca18b,8385080b-1cc6-44c5-a89b-85b8244591de
|
||||
1795180350.67535,warmup,201,0.01176905632019043,ae155b69f38b40efa06dc335484ce719,5d981da2-e2b0-4f58-945b-4e4f4fecd654
|
||||
1795180350.928941,warmup,201,0.020803213119506836,27cf380a14bf461eb619d155d979b8f4,8c4a855a-39a7-4ef8-bbbd-90e03768e88c
|
||||
1795180351.17329,warmup,201,0.018284082412719727,45247b8b0c484db4a857c5575a1da514,60f682a1-f799-4439-8f9a-082cbe158558
|
||||
1795180351.423593,warmup,201,0.02286505699157715,140ac94fb75846ccbc27e9e4dff67afa,f854fece-effa-4ab5-974b-72f86703bcac
|
||||
1795180351.678071,warmup,201,0.024228811264038086,e1ab03f2591b4971a7f90801486f2bf2,609aa3e0-7e44-496f-b64c-bbc1f9fa99a9
|
||||
1795180351.923866,warmup,201,0.01363992691040039,88c05ce549ef41c5a3a14c95c53bd409,365710a3-5f35-4adc-9c25-af3fe51493dd
|
||||
1795180352.177956,warmup,201,0.03061199188232422,21889320216743caa5c1392d19903af3,43dbfb78-3ebe-407b-b7e1-63e8cb3b9b5a
|
||||
1795180352.429983,warmup,201,0.012333154678344727,9de8eb1a06574abb826292e19b84f988,92e9a6bd-88fa-4610-b6c0-b5349308b4b0
|
||||
1795180352.674007,warmup,201,0.01972508430480957,f0ce8192797a4645b76b7e5808f2192f,6b22227e-67c4-4a9a-a0ca-08a8ef20e693
|
||||
1795180352.930673,warmup,201,0.014127016067504883,d3157bed5a4346baaec4b82e25019ecd,358d7a14-0c3d-45d2-be95-0630c878db02
|
||||
1795180353.172828,warmup,201,0.016306161880493164,48bb51fe3a604a499d067faf39701e9b,6ab03add-9f65-4731-a343-d154bf2a40e9
|
||||
1795180353.428106,warmup,201,0.021203994750976562,1583b7683f8141df8bb9883c0a1f189d,d3aa43a2-e707-41c2-8f1a-e3e1e7634623
|
||||
1795180353.677095,warmup,201,0.015395164489746094,d222f39f6a9a4d2e8f540d9b27aad707,320dfd89-7d56-42b7-b01c-cf04d8b513cd
|
||||
1795180353.9265192,warmup,201,0.08537173271179199,6a936c0664b74374a9f2187a2ddbbf06,708acef1-ae2b-4229-b4cc-9d19a2a9c57e
|
||||
1795180354.1799161,warmup,201,0.02597975730895996,c6bd819ca872476e94666e03b7f3992c,b3877af3-258b-4c77-8172-72b0bbee5b27
|
||||
1795180354.423289,warmup,201,0.010989904403686523,d30989f1247f4de7bfd4b9c32888d35c,09d175b4-c5be-4ded-bdea-47fee06baceb
|
||||
1795180354.674009,warmup,201,0.07170987129211426,1d4784d0947541f897b9279180f8d08f,2aac78f9-62f4-4f09-8d2d-526b9b21c9b0
|
||||
1795180354.924747,warmup,201,0.11100912094116211,ae9f5b183ecb4a67a1e29e88ecd685ca,837b2f21-0baa-4fc1-bd69-e53168ec033b
|
||||
1795180355.171475,warmup,201,0.181365966796875,929df48dbefa466dad31befb9d377330,e47a3bea-d950-47df-ac26-483c12bafe44
|
||||
1795180355.4301581,warmup,201,0.1972649097442627,275af00105a34802986c50739697c1ac,52788b1c-9221-4af9-8684-5b55a7aaf92a
|
||||
1795180355.673067,warmup,201,0.08739495277404785,39a13ad72b4448f0a767afb20028d7fc,59aa1c2d-b9f9-4634-af5c-c1977cb58871
|
||||
1795180355.92431,warmup,201,0.04182100296020508,a5a63d8af2ae4bedb575dc8739f3af2f,f5b3aac5-9941-4e03-a70e-e6e5e522a110
|
||||
1795180356.179839,warmup,201,0.06274628639221191,46d5eeb65b6d45ef9a81ec1369b7dd39,edbe7d82-fd87-423a-b49e-e395f740eb2c
|
||||
1795180356.427257,warmup,201,0.014807939529418945,f2d123f7377c4b0cbd1649a3785a2b90,ce02ec3a-c8a8-4123-a1ad-cc29835e8825
|
||||
1795180356.674825,warmup,201,0.011126041412353516,679d40c9ee6c481c81b28ce99f0e42b6,6c636c07-672e-4a2a-9058-db745c3463f0
|
||||
1795180356.927894,warmup,201,0.018886804580688477,552981d06df34d508ebd5846bb836a83,ffd07a80-f592-4ddc-a3cc-ff2a9f197fb6
|
||||
1795180357.1745431,warmup,201,0.009415864944458008,f79131e7537e4f0f8350b6fdcb0b96da,bec88e0d-4f2e-44b4-8b37-91062206723c
|
||||
1795180357.426614,warmup,201,0.04293084144592285,ae7be590ae294f2dac57e987ede2b4f6,df77674e-3378-4778-97d7-c89c7b2fa738
|
||||
1795180357.680987,warmup,201,0.12665009498596191,4b4003e700d546d1a66698887fa1380c,8a809c0b-b8cb-4074-ac85-8e7a43dca87e
|
||||
1795180357.9307442,warmup,201,0.022862672805786133,4fb7f73760a445dbac480366c1f672e5,6e063f18-7f51-4b19-8ed0-a0a7d73f95aa
|
||||
1795180358.1806982,warmup,201,0.037358760833740234,57b36f91a50a4302b67b697e8f8f08a4,0e1ea419-593b-4ca3-ac11-723e02b24ff3
|
||||
1795180358.423149,warmup,201,0.029945850372314453,603c36cdc55144618038a00cec0e2aec,e6102a10-97ca-432b-beb3-fa9f02501717
|
||||
1795180358.680745,warmup,201,0.01798725128173828,87ae437cafe647e29098811ddd6df769,b769fee9-17a2-49af-9d99-ac04d0201df4
|
||||
1795180358.921466,warmup,201,0.015487909317016602,f8c239845d61458d975549317f54a085,7e568898-e1ff-436c-8c80-673f25a8db5f
|
||||
1795180359.174216,warmup,201,0.016328811645507812,b052f222a93f4b7fbdef7bc74f89f2d2,bbf7df75-90b7-4ec7-abef-10d8ccce8c6c
|
||||
1795180359.4299269,warmup,201,0.0165250301361084,25510d762ac3496ba3eeb873580d0df5,9287de57-3881-4e80-93fd-7d2e550cab75
|
||||
1795180359.674637,warmup,201,0.018573760986328125,a4a36dd2e72d41c5af7dac81276b78f3,474e577f-89a4-485f-ae33-43dc9e0073b2
|
||||
1795180359.928923,warmup,201,0.036789894104003906,c48bf0ba1acb4dfeb92a33a5f7c6cf31,780b5f8d-4b0f-4ec5-ae82-f21d1727f87a
|
||||
1795180360.178257,warmup,201,0.2533531188964844,49c5d7b7dfc448e98f41e786cbdd2998,a0474355-8589-42f4-9094-7d8163b02b12
|
||||
1795180360.421695,warmup,201,0.052854061126708984,ff183e56bb5c4ac18f6d6f3859cc5100,0d099a99-b8b0-467d-91b4-e608e2d8f1d4
|
||||
1795180360.670847,warmup,201,0.06912517547607422,d40d667c826e49a8840ba9184578117c,bb64c924-9ff3-43d9-b2b2-4441497f8878
|
||||
1795180360.928828,warmup,201,0.08232808113098145,840f9a5f453249ba8c47882f57488674,8adf8c34-21aa-4000-829f-3775b3b5bcbf
|
||||
1795180361.1724942,warmup,201,0.17403888702392578,8c867b70da9649899232fe5772fc5df3,69a8b185-8eec-46fc-9c3c-1d857076d989
|
||||
1795180361.427998,warmup,201,0.021191835403442383,250cf8f6e9d54242abd898c562c6b5c9,67dae0d0-333d-4f9a-b909-014e5396107d
|
||||
1795180361.678324,warmup,201,0.024866104125976562,4d702fdc4a814531844a755099df5f46,2de43ad9-4cfc-40a3-a168-069a77fbe13f
|
||||
|
@@ -0,0 +1,373 @@
|
||||
{
|
||||
"date": "2026-11-20T13:11:41.242140+00:00",
|
||||
"source_commit": "0fc0f0f6200476c7295ad40540aae70b7673b9a8",
|
||||
"scope": "full30m",
|
||||
"profile": {
|
||||
"messages_per_second": 4,
|
||||
"max_inflight": 16,
|
||||
"upload_every_seconds": 5,
|
||||
"image_pixels": [
|
||||
320,
|
||||
200
|
||||
]
|
||||
},
|
||||
"complete": false,
|
||||
"samples": [
|
||||
{
|
||||
"at": 1795180312.8492448,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 0.0,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 31.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": null,
|
||||
"slo_good": 37.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 8901.0,
|
||||
"collector_sent_spans": 8886.0,
|
||||
"collector_received_logs": 731.0,
|
||||
"collector_sent_logs": 730.0,
|
||||
"collector_failed_spans": null,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 244039680.0,
|
||||
"app_memory": 657649664.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
{
|
||||
"at": 1795180322.153284,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 1.124622,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 32.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.24803624999999968,
|
||||
"slo_good": 38.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 9294.0,
|
||||
"collector_sent_spans": 9264.0,
|
||||
"collector_received_logs": 780.0,
|
||||
"collector_sent_logs": 776.0,
|
||||
"collector_failed_spans": null,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 244039680.0,
|
||||
"app_memory": 442171392.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
{
|
||||
"at": 1795180331.758197,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 1.454031,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 34.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.09944108981212381,
|
||||
"slo_good": 40.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 9685.0,
|
||||
"collector_sent_spans": 9667.0,
|
||||
"collector_received_logs": 830.0,
|
||||
"collector_sent_logs": 826.0,
|
||||
"collector_failed_spans": null,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 244047872.0,
|
||||
"app_memory": 442163200.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
{
|
||||
"at": 1795180340.560481,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 2.005743,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 36.0,
|
||||
"pool_used": 1.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.12987812499999915,
|
||||
"slo_good": 42.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 10067.0,
|
||||
"collector_sent_spans": 10055.0,
|
||||
"collector_received_logs": 876.0,
|
||||
"collector_sent_logs": 876.0,
|
||||
"collector_failed_spans": null,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 244133888.0,
|
||||
"app_memory": 442146816.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
{
|
||||
"at": 1795180349.6539578,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 0.831534,
|
||||
"worker_busy": 0.0,
|
||||
"worker_attempts": 38.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.09861986561062483,
|
||||
"slo_good": 44.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 10477.0,
|
||||
"collector_sent_spans": 10461.0,
|
||||
"collector_received_logs": 930.0,
|
||||
"collector_sent_logs": 928.0,
|
||||
"collector_failed_spans": null,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 244137984.0,
|
||||
"app_memory": 442433536.0
|
||||
},
|
||||
"alerts": []
|
||||
},
|
||||
{
|
||||
"at": 1795180359.34921,
|
||||
"phase": "warmup",
|
||||
"signals": {
|
||||
"api_error_ratio_1m": 0.0,
|
||||
"queue_depth": 0.0,
|
||||
"oldest_job_age": 1.390674,
|
||||
"worker_busy": 1.0,
|
||||
"worker_attempts": 39.0,
|
||||
"pool_used": 0.0,
|
||||
"pool_waiting": 0.0,
|
||||
"http_p95": 0.09654615268700784,
|
||||
"slo_good": 46.0,
|
||||
"slo_bad": 1.0,
|
||||
"collector_received_spans": 10867.0,
|
||||
"collector_sent_spans": 10855.0,
|
||||
"collector_received_logs": 977.0,
|
||||
"collector_sent_logs": 977.0,
|
||||
"collector_failed_spans": null,
|
||||
"collector_queue": 0.0,
|
||||
"collector_queue_capacity": 64.0,
|
||||
"collector_memory": 244211712.0,
|
||||
"app_memory": 442437632.0
|
||||
},
|
||||
"alerts": []
|
||||
}
|
||||
],
|
||||
"phases": [
|
||||
{
|
||||
"name": "warmup",
|
||||
"start": 1795180306.4255412,
|
||||
"planned_seconds": 300
|
||||
}
|
||||
],
|
||||
"container_limits": {
|
||||
"api": {
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "805306368",
|
||||
"memswap_limit": "805306368",
|
||||
"scale": 2
|
||||
},
|
||||
"broker": {
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "268435456",
|
||||
"memswap_limit": "268435456"
|
||||
},
|
||||
"cache": {
|
||||
"cpus": 0.25,
|
||||
"mem_limit": "134217728",
|
||||
"memswap_limit": "134217728"
|
||||
},
|
||||
"controller": {
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "201326592",
|
||||
"memswap_limit": "201326592"
|
||||
},
|
||||
"grafana": {
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "402653184",
|
||||
"memswap_limit": "402653184"
|
||||
},
|
||||
"lb": {
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "134217728",
|
||||
"memswap_limit": "134217728"
|
||||
},
|
||||
"loki": {
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "402653184",
|
||||
"memswap_limit": "402653184"
|
||||
},
|
||||
"migrate": {
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "268435456"
|
||||
},
|
||||
"observer": {
|
||||
"cpus": 0.25,
|
||||
"mem_limit": "201326592",
|
||||
"memswap_limit": "201326592"
|
||||
},
|
||||
"otel": {
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "268435456",
|
||||
"memswap_limit": "268435456"
|
||||
},
|
||||
"postgres": {
|
||||
"cpus": 1,
|
||||
"mem_limit": "536870912",
|
||||
"memswap_limit": "536870912"
|
||||
},
|
||||
"prometheus": {
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "402653184",
|
||||
"memswap_limit": "402653184"
|
||||
},
|
||||
"reconciler": {
|
||||
"cpus": 0.25,
|
||||
"mem_limit": "268435456",
|
||||
"memswap_limit": "268435456"
|
||||
},
|
||||
"relay": {
|
||||
"cpus": 0.25,
|
||||
"mem_limit": "201326592",
|
||||
"memswap_limit": "201326592"
|
||||
},
|
||||
"s3": {
|
||||
"cpus": 1,
|
||||
"mem_limit": "1073741824",
|
||||
"memswap_limit": "1073741824"
|
||||
},
|
||||
"s3-bootstrap": {
|
||||
"cpus": 0.25,
|
||||
"mem_limit": "134217728"
|
||||
},
|
||||
"telemetry-init": {
|
||||
"cpus": 0.1,
|
||||
"mem_limit": "67108864"
|
||||
},
|
||||
"tempo": {
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "536870912",
|
||||
"memswap_limit": "536870912"
|
||||
},
|
||||
"worker": {
|
||||
"cpus": 2,
|
||||
"mem_limit": "1610612736",
|
||||
"memswap_limit": "1610612736"
|
||||
}
|
||||
},
|
||||
"docker_vm": {
|
||||
"cpus": 8,
|
||||
"memory_bytes": 6209978368
|
||||
},
|
||||
"runtime_versions": [
|
||||
"752a5e6c0d33c2856f7ac8e66074e4adfb9a699d"
|
||||
],
|
||||
"elapsed_seconds": 60.619,
|
||||
"requests_csv": "slo-research-final-requests.csv",
|
||||
"request_statuses": {
|
||||
"201": 222
|
||||
},
|
||||
"uploads": [
|
||||
[
|
||||
1795180306.424507,
|
||||
"warmup",
|
||||
202,
|
||||
"2673e3f4-4b94-4765-a221-82a19932f481",
|
||||
"9b97204e2328468687d4696df121341f"
|
||||
],
|
||||
[
|
||||
1795180311.425008,
|
||||
"warmup",
|
||||
202,
|
||||
"70330a66-051f-46b2-af95-ba3676a59c24",
|
||||
"0f7d5b83e7c546dea644259dfa669c48"
|
||||
],
|
||||
[
|
||||
1795180316.428648,
|
||||
"warmup",
|
||||
202,
|
||||
"5ba1e41c-a3c0-4e3a-8773-8d149259cb0c",
|
||||
"5e4502be41b549e6ab5573395740c823"
|
||||
],
|
||||
[
|
||||
1795180321.436966,
|
||||
"warmup",
|
||||
202,
|
||||
"16818c1b-cdd1-4a85-ac31-ebd86187e72d",
|
||||
"afdbfb18874b48c1a0ceeacbda2deebf"
|
||||
],
|
||||
[
|
||||
1795180326.444726,
|
||||
"warmup",
|
||||
202,
|
||||
"4ded2fa3-4426-4e5d-9f83-f36050749600",
|
||||
"ff3acf4079b342d087ad72006c4f8769"
|
||||
],
|
||||
[
|
||||
1795180331.447099,
|
||||
"warmup",
|
||||
202,
|
||||
"0e5adf2b-4b27-4efd-a0e0-46f81733df47",
|
||||
"daeebdbd18e24af58d20a629d2b4119b"
|
||||
],
|
||||
[
|
||||
1795180336.4565601,
|
||||
"warmup",
|
||||
202,
|
||||
"e0c53652-5dc1-44f9-ad0d-35863f5af1c9",
|
||||
"c2d706ff5e3d488a91d008644719b015"
|
||||
],
|
||||
[
|
||||
1795180341.461115,
|
||||
"warmup",
|
||||
202,
|
||||
"82d2a98d-69a6-4148-83de-81c85fc3f2c1",
|
||||
"cea48c7bc3d7422998ede64dbd5f6106"
|
||||
],
|
||||
[
|
||||
1795180346.469613,
|
||||
"warmup",
|
||||
202,
|
||||
"28b5f6b2-a7b9-428c-9091-2a001de0dfca",
|
||||
"2e52c5b0c8b84eca9c10f1955a72a20f"
|
||||
],
|
||||
[
|
||||
1795180351.475091,
|
||||
"warmup",
|
||||
202,
|
||||
"ec5287c6-2bd1-4de7-9ea4-ed4a4bb94026",
|
||||
"ec278024a9c24ad193b8ff500bb0bfd4"
|
||||
],
|
||||
[
|
||||
1795180356.482144,
|
||||
"warmup",
|
||||
202,
|
||||
"1efafb25-a1bd-469f-bebc-e3fac287b86b",
|
||||
"cbacdf851ac74a5c9fbed2c8c78739f0"
|
||||
],
|
||||
[
|
||||
1795180361.482274,
|
||||
"warmup",
|
||||
202,
|
||||
"d951a0c4-78d0-4a2c-8bfa-4880fb0bd9f6",
|
||||
"c5675965b42e4644a1112505c3a83816"
|
||||
]
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,107 @@
|
||||
// Fetch the immutable module ZIP in bounded HTTP ranges. The subsequent
|
||||
// go mod download validates its h1 checksum through the Go checksum database.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
const version = "v0.0.0-20251015172955-9e49d5e7a648"
|
||||
const total int64 = 25342606
|
||||
const chunk int64 = 512 * 1024
|
||||
const source = "https://proxy.golang.org/github.com/minio/minio/@v/" + version + ".zip"
|
||||
directory := "/go/pkg/mod/cache/download/github.com/minio/minio/@v"
|
||||
if err := os.MkdirAll(directory, 0755); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
target := filepath.Join(directory, version+".zip")
|
||||
if info, err := os.Stat(target); err == nil && info.Size() == total {
|
||||
fmt.Println("Source ZIP cached; Go will validate checksum")
|
||||
return
|
||||
}
|
||||
client := &http.Client{Timeout: 90 * time.Second}
|
||||
jobs := make(chan int64)
|
||||
errors := make(chan error, 64)
|
||||
var group sync.WaitGroup
|
||||
for worker := 0; worker < 8; worker++ {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
for start := range jobs {
|
||||
end := min(start+chunk, total) - 1
|
||||
part := fmt.Sprintf("%s.part-%d", target, start)
|
||||
if info, err := os.Stat(part); err == nil && info.Size() == end-start+1 {
|
||||
continue
|
||||
}
|
||||
var last error
|
||||
for attempt := 0; attempt < 4; attempt++ {
|
||||
req, _ := http.NewRequest("GET", source, nil)
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
last = fmt.Errorf("source range request failed")
|
||||
continue
|
||||
}
|
||||
expected := fmt.Sprintf("bytes %d-%d/%d", start, end, total)
|
||||
if response.StatusCode != 206 || response.Header.Get("Content-Range") != expected {
|
||||
response.Body.Close()
|
||||
last = fmt.Errorf("unexpected source range response: HTTP %d", response.StatusCode)
|
||||
continue
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(response.Body, end-start+2))
|
||||
response.Body.Close()
|
||||
if err != nil || int64(len(data)) != end-start+1 {
|
||||
last = fmt.Errorf("incomplete source range")
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(part, data, 0644); err != nil {
|
||||
last = err
|
||||
continue
|
||||
}
|
||||
last = nil
|
||||
fmt.Printf("Source part %d verified\n", start/chunk+1)
|
||||
break
|
||||
}
|
||||
if last != nil {
|
||||
errors <- last
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for start := int64(0); start < total; start += chunk {
|
||||
jobs <- start
|
||||
}
|
||||
close(jobs)
|
||||
group.Wait()
|
||||
close(errors)
|
||||
for err := range errors {
|
||||
panic(err)
|
||||
}
|
||||
out, err := os.Create(target + ".partial")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for start := int64(0); start < total; start += chunk {
|
||||
part, err := os.Open(fmt.Sprintf("%s.part-%d", target, start))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, err := io.Copy(out, part); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
part.Close()
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := os.Rename(target+".partial", target); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,789 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Мессенджер: диагностика и SLO",
|
||||
"schemaVersion": 41,
|
||||
"version": 1,
|
||||
"editable": false,
|
||||
"timezone": "utc",
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-30m",
|
||||
"to": "now"
|
||||
},
|
||||
"tags": [
|
||||
"lab6",
|
||||
"slo"
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Запросы через балансировщик, в секунду",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\"}[1m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "все бизнес-запросы",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "Health requests use a separate backend and are excluded. Includes ingress 503 with no API available.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Доля ответов 5xx",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\",code=\"5xx\"}[1m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\"}[1m])),0.001)",
|
||||
"refId": "A",
|
||||
"legendFormat": "ошибки 5xx",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API: p50 / p95 / p99, секунды",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.5, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "p50",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
|
||||
"refId": "B",
|
||||
"legendFormat": "p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
|
||||
"refId": "C",
|
||||
"legendFormat": "p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "Quantiles from combined histogram buckets; complete ASGI response, no average of replica quantiles.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Очередь: ожидающие задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(messenger_queue_depth)",
|
||||
"refId": "A",
|
||||
"legendFormat": "ожидают",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "Logical PostgreSQL jobs in queued state, not count of duplicate stream notifications.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Возраст старейшего незавершённого задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(messenger_oldest_job_age_seconds)",
|
||||
"refId": "A",
|
||||
"legendFormat": "возраст",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Обработчик: занято / доступно",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(messenger_worker_busy)",
|
||||
"refId": "A",
|
||||
"legendFormat": "занято",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "sum(messenger_worker_capacity)",
|
||||
"refId": "B",
|
||||
"legendFormat": "всего мест",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Соединения PostgreSQL",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(service_name) (messenger_db_pool_used)",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} занято",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "sum by(service_name) (messenger_db_pool_waiting)",
|
||||
"refId": "B",
|
||||
"legendFormat": "{{service_name}} ожидают",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Попытки обработки по результату",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(outcome) (rate(messenger_job_attempts_total[2m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{outcome}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Длительность попытки обработки: p95",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_duration_seconds_bucket[2m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "От приёма до ready: p50 / p95 / p99",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.5,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "p50",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
|
||||
"refId": "B",
|
||||
"legendFormat": "p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
|
||||
"refId": "C",
|
||||
"legendFormat": "p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "Only completed publications have a duration. Deadline SLI below also counts unfinished tasks.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "SLO API за 30 минут: цель 99%",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - sum(increase(haproxy_backend_http_responses_total{proxy=\"messenger\",code=\"5xx\"}[30m])) / clamp_min(sum(increase(haproxy_backend_http_responses_total{proxy=\"messenger\"}[30m])),1)",
|
||||
"refId": "A",
|
||||
"legendFormat": "доступность",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "SLO ready за 30 минут: 99% не позже 10 секунд",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(messenger_slo_decisions_total{outcome=\"good\"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)",
|
||||
"refId": "A",
|
||||
"legendFormat": "доля вовремя",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "Durable one-time decisions per non-obsolete logical job; unfinished jobs become bad at ten seconds. Synthetic 320x200 profile.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "CPU процессов, ядра",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(service_name)(rate(messenger_process_cpu_seconds_total[1m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Память процессов",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max by(service_name)(messenger_process_memory_bytes)",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Буфер экспорта Collector",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "otelcol_exporter_queue_size",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "otelcol_exporter_queue_capacity",
|
||||
"refId": "B",
|
||||
"legendFormat": "{{exporter}} capacity",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Ошибки отправки трасс и логов",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_spans[1m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "spans/s",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_log_records[1m]))",
|
||||
"refId": "B",
|
||||
"legendFormat": "logs/s",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Состояния локальных alerts",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "ALERTS{alertstate=\"firing\"}",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{alertname}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"title": "Связанные логи API и обработчика",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
|
||||
"queryType": "range",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
}
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"wrapLogMessage": true,
|
||||
"sortOrder": "Descending"
|
||||
},
|
||||
"description": "Filter structured metadata trace_id in Explore; open the same ID in the Traces datasource."
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Messenger
|
||||
orgId: 1
|
||||
folder: Laboratory 6
|
||||
type: file
|
||||
disableDeletion: true
|
||||
editable: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,37 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Metrics
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
timeInterval: 5s
|
||||
- name: Logs
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
url: $${__value.raw}
|
||||
datasourceUid: tempo
|
||||
- name: Traces
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
spanStartTimeShift: -1m
|
||||
spanEndTimeShift: 1m
|
||||
@@ -0,0 +1,50 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 512
|
||||
stats socket /tmp/admin.sock mode 600 level admin
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
log-format "http status=%ST duration_ms=%Ta"
|
||||
timeout connect 1s
|
||||
timeout client 15s
|
||||
timeout server 8s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
use_backend health if { path_beg /health/ }
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 250ms fall 2 rise 1 resolvers docker init-addr libc,none resolve-prefer ipv4
|
||||
server-template api 1-8 api:8080
|
||||
|
||||
backend health
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 1 resolvers docker init-addr libc,none
|
||||
server-template api 1-8 api:8080
|
||||
|
||||
frontend internal_metrics
|
||||
bind :8404
|
||||
http-request use-service prometheus-exporter if { path /metrics }
|
||||
http-request deny
|
||||
@@ -1,7 +1,9 @@
|
||||
# Конфигурация наблюдаемости — реализовать
|
||||
# Конфигурация наблюдаемости
|
||||
|
||||
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
|
||||
`collector.yaml`: OTLP/HTTP4318 внутри Compose, memory limiter/batch, три pipelines. Prometheus exporter8889 для приложения, self metrics8888. Log pipeline отправляется в Loki native OTLP, trace pipeline — в Tempo. Конечные очереди и повторные отправки сохраняют ограничение RAM при отказах. Эти ports на host не публикуются.
|
||||
|
||||
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
|
||||
`prometheus.yaml`: scrape/evaluation5с, приложение/Collector/HAProxy/Tempo/Loki. `alerts.yaml`: API/ready multi-window burn и предупреждения о5xx/возрасте очереди. [RUNBOOK](RUNBOOK.md) содержит шаги диагностики и определения SLO.
|
||||
|
||||
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
|
||||
`loki.yaml` и `tempo.yaml`: локальные persistent storage/WAL, provisioning Grafana находится в `infra/grafana`. Dashboard JSON импортируется автоматически, uid messenger-lab6,18панелей/3datasources. Версии/digests закреплены в Compose.
|
||||
|
||||
Результаты: [REPORT](../../REPORT.md), causal/privacy/Grafana proof `evidence/telemetry-first.json`, полный30минутный опыт `evidence/slo-research-verified.json`, анализ/график `evidence/slo-analysis-verified/`. Неполные предыдущие опыты сохранены с complete=false и не заменяют успешный результат. Отдельный clean-clone receipt должен подтвердить перезапуск и сохранение истории каждого backend.
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# Диагностика по сохранённой телеметрии
|
||||
|
||||
Открой dashboard «Мессенджер: диагностика и SLO», временная зона UTC.
|
||||
Сопоставляй время оповещения с метриками, трассами и связанными логами.
|
||||
|
||||
## API errors
|
||||
|
||||
Признак: ApiHighErrorRate, ApiFastBurn или ApiSlowBurn. Числитель доступности:
|
||||
`sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m]))`.
|
||||
Знаменатель — тот же backend со всеми code. Health использует другой backend.
|
||||
Сравни API histogram, DB pool used/waiting и worker busy. В Traces найди
|
||||
ошибочный server span и вложенный db.postgresql или s3.*. Длинная DB-операция
|
||||
с ошибкой и занятым пулом при нормальной очереди указывает на БД. Отсутствие
|
||||
API span при ingress503 может означать отсутствие готовых API. Один5xx не
|
||||
доказывает причину. Через trace_id найди соответствующий Logs record.
|
||||
После исправления проверь новые сообщения, спад5xx, resolved и прежние данные.
|
||||
|
||||
## Queue age
|
||||
|
||||
Признак: QueueLagGrowing, ReadyFastBurn или ReadySlowBurn. Сравни логические
|
||||
queued jobs PG, oldest job age и worker busy/capacity. Upload202 при росте
|
||||
ожидающей очереди и отсутствии новых worker spans указывает на обработчик.
|
||||
Если попытки идут и содержат DB/S3 ошибки, причина может быть в зависимости.
|
||||
Найди путь upload→producer→consumer→S3/DB. Histogram ready содержит только
|
||||
завершения, поэтому отдельно проверь deadline SLI для незавершённых jobs.
|
||||
После восстановления проверь тот же attachment_id, original, спад и resolved.
|
||||
|
||||
## Telemetry backend
|
||||
|
||||
Проверь backend health, Collector queue и accepted/sent/failed счётчики.
|
||||
Failed attempts не равны числу уникально потерянных spans: пакет повторяется.
|
||||
После слива очередей сравни изменение accepted-minus-sent и исходное состояние.
|
||||
SDK queues traces/logs512records, timeout0.75s; Collector memory limiter192MiB,
|
||||
batch≤256, очереди logs/traces по32пакета, один sender и finite retry≤10s.
|
||||
При отказе telemetry может теряться; измеряй память и настоящие бизнес-запросы.
|
||||
После восстановления найди новую трассу и лог, а не только старый screenshot.
|
||||
|
||||
## SLO и окна
|
||||
|
||||
Учебный профиль: синтетические PNG320×200, upload каждые5s,
|
||||
закреплённые CPU/RAM. API цель99%ответов без5xx за30m на ingress. Jobs цель99%
|
||||
ready≤10s от PG created_at принимающей транзакции до committed_at публикации.
|
||||
Начальная граница консервативно включает завершение принимающей транзакции.
|
||||
Однократные durable good/bad решения учитывают failed и незавершённые после10s.
|
||||
Ручной replay не стирает прежний bad; устаревшие до решения jobs исключены.
|
||||
Burn rate = доля плохих событий/0.01. Быстрое правило: burn>14.4 в30s и2m,
|
||||
непрерывно45s. Длительное: burn>6 в2m и5m, непрерывно60s. Короткие окна выбраны
|
||||
для воспроизводимого стенда, а не универсальной production paging policy.
|
||||
Измеряются краткий всплеск/длительная деградация/firing/resolved/ложные события.
|
||||
Точность времени ограничена scrape/evaluation5s, экспортом и реальным polling: в полном опыте медиана9.663s, максимум38.223s.
|
||||
@@ -0,0 +1,53 @@
|
||||
groups:
|
||||
- name: messenger-slo
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: ApiHighErrorRate
|
||||
expr: sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) > 0.05
|
||||
for: 20s
|
||||
labels: {severity: warning, signal: api}
|
||||
annotations:
|
||||
summary: More than 5 percent business responses are 5xx
|
||||
runbook: infra/otel/RUNBOOK.md#api-errors
|
||||
- alert: QueueLagGrowing
|
||||
expr: max(messenger_oldest_job_age_seconds) > 20 and max(messenger_queue_depth) > 0
|
||||
for: 10s
|
||||
labels: {severity: warning, signal: ready}
|
||||
annotations:
|
||||
summary: Oldest unfinished attachment exceeds 20 seconds
|
||||
runbook: infra/otel/RUNBOOK.md#queue-age
|
||||
- alert: ApiFastBurn
|
||||
expr: (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[30s])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[30s])),0.001) / 0.01 > 14.4) and (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) / 0.01 > 14.4)
|
||||
for: 45s
|
||||
labels: {severity: critical, signal: api, burn: fast}
|
||||
annotations:
|
||||
summary: API burns the one-percent error budget in both fast windows
|
||||
runbook: infra/otel/RUNBOOK.md#api-errors
|
||||
- alert: ApiSlowBurn
|
||||
expr: (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) / 0.01 > 6) and (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[5m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[5m])),0.001) / 0.01 > 6)
|
||||
for: 60s
|
||||
labels: {severity: critical, signal: api, burn: slow}
|
||||
annotations:
|
||||
summary: API burns the error budget across two longer windows
|
||||
runbook: infra/otel/RUNBOOK.md#api-errors
|
||||
- alert: ReadyFastBurn
|
||||
expr: (sum(rate(messenger_slo_decisions_total{outcome="bad"}[30s])) / clamp_min(sum(rate(messenger_slo_decisions_total[30s])),0.001) / 0.01 > 14.4) and (sum(rate(messenger_slo_decisions_total{outcome="bad"}[2m])) / clamp_min(sum(rate(messenger_slo_decisions_total[2m])),0.001) / 0.01 > 14.4)
|
||||
for: 45s
|
||||
labels: {severity: critical, signal: ready, burn: fast}
|
||||
annotations:
|
||||
summary: Attachment deadline SLO burns the error budget
|
||||
runbook: infra/otel/RUNBOOK.md#queue-age
|
||||
- alert: ReadySlowBurn
|
||||
expr: (sum(rate(messenger_slo_decisions_total{outcome="bad"}[2m])) / clamp_min(sum(rate(messenger_slo_decisions_total[2m])),0.001) / 0.01 > 6) and (sum(rate(messenger_slo_decisions_total{outcome="bad"}[5m])) / clamp_min(sum(rate(messenger_slo_decisions_total[5m])),0.001) / 0.01 > 6)
|
||||
for: 60s
|
||||
labels: {severity: critical, signal: ready, burn: slow}
|
||||
annotations:
|
||||
summary: Sustained misses of the attachment ten-second deadline
|
||||
runbook: infra/otel/RUNBOOK.md#queue-age
|
||||
- alert: CollectorUnavailable
|
||||
expr: up{job="collector"} == 0
|
||||
for: 15s
|
||||
labels: {severity: warning, signal: telemetry}
|
||||
annotations:
|
||||
summary: Collector self metrics are unavailable
|
||||
runbook: infra/otel/RUNBOOK.md#telemetry-backend
|
||||
@@ -0,0 +1,88 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
metric_expiration: 15s
|
||||
resource_constant_labels:
|
||||
included:
|
||||
- service.name
|
||||
- service.instance.id
|
||||
- service.version
|
||||
- deployment.environment.name
|
||||
otlp_http/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
queue_size: 32
|
||||
num_consumers: 1
|
||||
block_on_overflow: false
|
||||
retry_on_failure:
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 10s
|
||||
otlp_http/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
queue_size: 32
|
||||
num_consumers: 1
|
||||
block_on_overflow: false
|
||||
retry_on_failure:
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 10s
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
service:
|
||||
extensions:
|
||||
- health_check
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
level: detailed
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlp_http/tempo
|
||||
logs:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlp_http/loki
|
||||
metrics:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- prometheus
|
||||
@@ -0,0 +1,38 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
log_level: warn
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2026-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
ingestion_rate_mb: 4
|
||||
ingestion_burst_size_mb: 8
|
||||
volume_enabled: true
|
||||
otlp_config:
|
||||
resource_attributes:
|
||||
ignore_defaults: true
|
||||
attributes_config:
|
||||
- action: index_label
|
||||
attributes: [service.name]
|
||||
- action: structured_metadata
|
||||
attributes: [service.instance.id, service.version, deployment.environment.name]
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,23 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
rule_files: [/etc/prometheus/alerts.yaml]
|
||||
scrape_configs:
|
||||
- job_name: application_otlp
|
||||
static_configs:
|
||||
- targets: [otel:8889]
|
||||
- job_name: collector
|
||||
static_configs:
|
||||
- targets: [otel:8888]
|
||||
- job_name: ingress
|
||||
static_configs:
|
||||
- targets: [lb:8404]
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: [localhost:9090]
|
||||
- job_name: tempo
|
||||
static_configs:
|
||||
- targets: [tempo:3200]
|
||||
- job_name: loki
|
||||
static_configs:
|
||||
- targets: [loki:3100]
|
||||
@@ -0,0 +1,20 @@
|
||||
target: all
|
||||
stream_over_http_enabled: true
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
log_level: warn
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"otel": "otel/opentelemetry-collector-contrib:0.165.0@sha256:e40d61f10c9a617cca1ffd1be4d7e2b3a4b0f0001798ff07e58cabe82e618cea",
|
||||
"prometheus": "prom/prometheus:v3.16.0@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06",
|
||||
"loki": "grafana/loki:3.7.10@sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7",
|
||||
"tempo": "grafana/tempo:3.1.1@sha256:64e2fbd56d9c2b3412a4b430a58ea59e91a670b1e75f920a0c702213ad145044",
|
||||
"grafana": "grafana/grafana:13.3.1@sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf"
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
root_user=$(cat /run/secrets/s3_root_user)
|
||||
root_password=$(cat /run/secrets/s3_root_password)
|
||||
api_key=$(cat /run/secrets/s3_access_key)
|
||||
api_secret=$(cat /run/secrets/s3_secret_key)
|
||||
# Credentials are passed only inside this isolated bootstrap container.
|
||||
mc alias set local http://s3:9000 "$root_user" "$root_password" >/dev/null 2>&1
|
||||
mc mb --ignore-existing local/messenger >/dev/null 2>&1
|
||||
mc anonymous set none local/messenger >/dev/null 2>&1
|
||||
mc admin user add local "$api_key" "$api_secret" >/dev/null 2>&1
|
||||
mc admin policy create local messenger-api /policy.json >/dev/null 2>&1
|
||||
mc admin policy attach local messenger-api --user "$api_key" >/dev/null 2>&1
|
||||
printf '%s\n' 'Private bucket and scoped application policy ready'
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
|
||||
export MINIO_BROWSER_REDIRECT_URL=http://127.0.0.1:9001
|
||||
exec minio server /data --address :9000 --console-address 127.0.0.1:9001
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::messenger"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger/objects/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
FROM golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 AS toolchain
|
||||
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOFLAGS=-p=2 GOTOOLCHAIN=local
|
||||
|
||||
FROM toolchain AS minio-build
|
||||
WORKDIR /source
|
||||
COPY infra/fetch-source.go /tmp/fetch-source.go
|
||||
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
|
||||
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
|
||||
go run /tmp/fetch-source.go && \
|
||||
for attempt in 1 2 3 4; do \
|
||||
go mod download -json github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648 && break; \
|
||||
test "$attempt" -lt 4 || exit 1; sleep 1; \
|
||||
done && \
|
||||
cp -a /go/pkg/mod/github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648/. /source/
|
||||
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
|
||||
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
|
||||
for attempt in 1 2 3; do \
|
||||
timeout 900s go build -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' -o /out/minio . && break; \
|
||||
test "$attempt" -lt 3 || exit 1; sleep 1; \
|
||||
done
|
||||
|
||||
FROM toolchain AS mc-build
|
||||
WORKDIR /source
|
||||
RUN git init . && git remote add origin https://github.com/minio/mc.git && \
|
||||
git fetch --depth 1 origin 7394ce0dd2a80935aded936b09fa12cbb3cb8096 && git checkout --detach FETCH_HEAD
|
||||
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
|
||||
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
|
||||
go build -trimpath -ldflags='-s -w' -o /out/mc .
|
||||
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS server
|
||||
LABEL org.opencontainers.image.revision=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z
|
||||
COPY --from=minio-build /out/minio /usr/local/bin/minio
|
||||
COPY --from=minio-build /source/LICENSE /licenses/minio.LICENSE
|
||||
COPY --chmod=755 infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint
|
||||
RUN mkdir /data && chown 10001:10001 /data
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/local/bin/s3-entrypoint"]
|
||||
EXPOSE 9000
|
||||
HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=30 CMD ["python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready',timeout=1).read()"]
|
||||
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS bootstrap
|
||||
LABEL org.opencontainers.image.revision=7394ce0dd2a80935aded936b09fa12cbb3cb8096 org.opencontainers.image.version=RELEASE.2025-08-13T08-35-41Z
|
||||
COPY --from=mc-build /out/mc /usr/local/bin/mc
|
||||
COPY --from=mc-build /source/LICENSE /licenses/mc.LICENSE
|
||||
COPY --chmod=755 infra/s3-bootstrap.sh /usr/local/bin/s3-bootstrap
|
||||
COPY infra/s3-policy.json /policy.json
|
||||
USER 10001:10001
|
||||
ENV MC_CONFIG_DIR=/tmp/mc
|
||||
ENTRYPOINT ["/usr/local/bin/s3-bootstrap"]
|
||||
@@ -0,0 +1,36 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position integer NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
CREATE INDEX members_user_chat_idx ON members(user_id,chat_id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL,
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
|
||||
CREATE TABLE sessions (
|
||||
token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
created_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
body_hash bytea NOT NULL CHECK (octet_length(body_hash)=32),
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY(user_id, chat_id, key)
|
||||
);
|
||||
CREATE INDEX idempotency_expiry_idx ON idempotency(expires_at);
|
||||
@@ -0,0 +1,32 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
|
||||
ALTER TABLE messages ADD UNIQUE(chat_id,id);
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL,
|
||||
filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
state text NOT NULL CHECK(state IN ('pending','ready','failed')),
|
||||
error_code text,
|
||||
uploader_instance text NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE(chat_id,id),
|
||||
FOREIGN KEY(chat_id,owner_id) REFERENCES members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX attachments_state_created_idx ON attachments(state,created_at,id);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position integer NOT NULL CHECK(position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY(message_id,attachment_id),
|
||||
UNIQUE(message_id,position),
|
||||
FOREIGN KEY(chat_id,message_id) REFERENCES messages(chat_id,id),
|
||||
FOREIGN KEY(chat_id,attachment_id) REFERENCES attachments(chat_id,id)
|
||||
);
|
||||
@@ -0,0 +1,54 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
|
||||
ALTER TABLE attachments ADD CHECK(state IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>=1);
|
||||
ALTER TABLE attachments ADD COLUMN published_version integer;
|
||||
CREATE TABLE upload_reservations (
|
||||
id uuid PRIMARY KEY, object_key text UNIQUE NOT NULL,
|
||||
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds'
|
||||
);
|
||||
CREATE TABLE jobs (
|
||||
id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL CHECK(pipeline_version>=1), source_key text NOT NULL,
|
||||
state text NOT NULL CHECK(state IN ('queued','processing','ready','failed','obsolete')),
|
||||
attempt integer NOT NULL DEFAULT 0, cycle integer NOT NULL DEFAULT 1,
|
||||
fence bigint NOT NULL DEFAULT 0, lease_owner text, lease_until timestamptz,
|
||||
next_at timestamptz NOT NULL DEFAULT clock_timestamp(), error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE(attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE INDEX jobs_due_idx ON jobs(state,next_at);
|
||||
CREATE TABLE outbox (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id),
|
||||
payload jsonb NOT NULL, due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz
|
||||
);
|
||||
CREATE INDEX outbox_due_idx ON outbox(due_at,id) WHERE published_at IS NULL;
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
|
||||
name text NOT NULL CHECK(name IN ('original','thumbnail','crop','optimized')),
|
||||
object_key text UNIQUE NOT NULL, content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL, sha256 text NOT NULL,
|
||||
width integer, height integer,
|
||||
PRIMARY KEY(attachment_id,pipeline_version,name)
|
||||
);
|
||||
CREATE TABLE effect_publications (
|
||||
attachment_id uuid NOT NULL, pipeline_version integer NOT NULL,
|
||||
job_id uuid NOT NULL REFERENCES jobs(id), fence bigint NOT NULL,
|
||||
committed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY(attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE TABLE pipeline_audit (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL, action text NOT NULL,
|
||||
operator text NOT NULL, reason text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
-- Fault injection is disabled unless PIPELINE_FAULTS=1; no public endpoint.
|
||||
CREATE TABLE pipeline_faults (
|
||||
id bigserial PRIMARY KEY, target uuid NOT NULL, phase text NOT NULL,
|
||||
action text NOT NULL CHECK(action IN ('wait','crash','transient','permanent','hang')),
|
||||
remaining integer NOT NULL DEFAULT 1, hit_at timestamptz, released_at timestamptz
|
||||
);
|
||||
CREATE TABLE pipeline_events (
|
||||
id bigserial PRIMARY KEY, job_id uuid, event text NOT NULL,
|
||||
detail jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
@@ -0,0 +1,10 @@
|
||||
ALTER TABLE jobs ADD COLUMN traceparent text NOT NULL DEFAULT '';
|
||||
ALTER TABLE jobs ADD COLUMN tracestate text NOT NULL DEFAULT '';
|
||||
CREATE TABLE job_slo_observations (
|
||||
job_id uuid PRIMARY KEY REFERENCES jobs(id),
|
||||
outcome text NOT NULL CHECK(outcome IN ('good','bad')),
|
||||
decided_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
histogram_epoch uuid
|
||||
);
|
||||
CREATE INDEX job_slo_epoch_idx ON job_slo_observations(histogram_epoch);
|
||||
CREATE INDEX effect_publications_time_idx ON effect_publications(committed_at,job_id);
|
||||
@@ -0,0 +1,58 @@
|
||||
[project]
|
||||
name = "mtusi-messenger-lab6"
|
||||
version = "1.0.0"
|
||||
requires-python = ">=3.14,<3.15"
|
||||
dependencies = [
|
||||
"fastapi==0.143.3",
|
||||
"pydantic==2.14.1",
|
||||
"uvicorn==0.54.0",
|
||||
"psycopg[binary,pool]==3.3.6",
|
||||
"argon2-cffi>=25,<26",
|
||||
"redis==8.1.0",
|
||||
"minio==7.2.20",
|
||||
"pillow==13.0.0",
|
||||
"opentelemetry-api==1.45.2",
|
||||
"opentelemetry-sdk==1.45.2",
|
||||
"opentelemetry-exporter-otlp-proto-http==1.45.2",
|
||||
"psutil==7.1.3",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8,<10",
|
||||
"httpx>=0.28,<1",
|
||||
"ruff>=0.14,<1",
|
||||
"openapi-spec-validator==0.7.2",
|
||||
"psutil>=7,<8",
|
||||
"matplotlib>=3.10,<4",
|
||||
"pyyaml==6.0.3",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 79
|
||||
target-version = "py314"
|
||||
include = ["src/**/*.py", "student-tests/**/*.py", "student-unit/**/*.py", "research/**/*.py", "scripts/**/*.py"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "B", "UP"]
|
||||
|
||||
[tool.ruff.per-file-target-version]
|
||||
"scripts/*.py" = "py39"
|
||||
|
||||
# SQL и генерируемые Markdown-таблицы в инструментах не разрываем.
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"research/*.py" = ["E501"]
|
||||
"scripts/*.py" = ["E501"]
|
||||
|
||||
# Keep SQL statements together; the formatter controls executable statements.
|
||||
"src/attachments.py" = ["E501"]
|
||||
"src/pipeline.py" = ["E501"]
|
||||
"src/worker.py" = ["E501"]
|
||||
"src/operator.py" = ["E501"]
|
||||
"src/reconcile.py" = ["E501"]
|
||||
"src/observer.py" = ["E501"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["student-tests"]
|
||||
addopts = "-ra"
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Summarize measured fault invariants and child resource samples."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def effects(case):
|
||||
for key in ["effect", "v1", "v2"]:
|
||||
if key in case:
|
||||
yield key, case[key]
|
||||
for i, item in enumerate(case.get("effects", [])):
|
||||
yield str(i + 1), item
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("input", type=Path)
|
||||
p.add_argument("--output", required=True, type=Path)
|
||||
a = p.parse_args()
|
||||
d = json.loads(a.input.read_text())
|
||||
assert d["complete"] and d.get("scope", "all") == "all"
|
||||
rows = []
|
||||
for case in d["cases"]:
|
||||
for role, e in effects(case):
|
||||
j = e["job"]
|
||||
rows.append(
|
||||
dict(
|
||||
case=case["case"],
|
||||
role=role,
|
||||
job_id=j["id"],
|
||||
version=j["pipeline_version"],
|
||||
attempt=j["attempt"],
|
||||
cycle=j["cycle"],
|
||||
fence=j["fence"],
|
||||
state=j["state"],
|
||||
publications=e["publications"],
|
||||
variant_rows=len(e["variants"]),
|
||||
physical_keys=len(e["physical_keys"]),
|
||||
)
|
||||
)
|
||||
a.output.mkdir(parents=True, exist_ok=True)
|
||||
with (a.output / "effects.csv").open("w") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
ready = [r for r in rows if r["state"] == "ready"]
|
||||
assert all(
|
||||
r["publications"] == 1
|
||||
and r["variant_rows"] == 4
|
||||
and r["physical_keys"] == 4
|
||||
for r in ready
|
||||
)
|
||||
stale = [r for r in rows if r["state"] == "obsolete"]
|
||||
assert all(r["publications"] == 0 for r in stale)
|
||||
job_ids = {r["job_id"] for r in rows}
|
||||
samples = [
|
||||
x["detail"]
|
||||
for x in d["worker_metrics"]
|
||||
if str(x["job_id"]) in job_ids
|
||||
and x["detail"].get("result") == "ready"
|
||||
and "rss_peak_mib" in x["detail"]
|
||||
]
|
||||
stats = {
|
||||
"source_commit": d["source_commit"],
|
||||
"cases": len(d["cases"]),
|
||||
"observed_ready_effects": len(ready),
|
||||
"observed_obsolete_effects": len(stale),
|
||||
"all_ready_have_one_publication_and_four_variants": True,
|
||||
"all_obsolete_have_zero_publications": True,
|
||||
"successful_child_samples": len(samples),
|
||||
"child_elapsed_range_seconds": [
|
||||
min(x["seconds"] for x in samples),
|
||||
max(x["seconds"] for x in samples),
|
||||
],
|
||||
"child_cpu_range_seconds": [
|
||||
min(x["cpu_seconds"] for x in samples),
|
||||
max(x["cpu_seconds"] for x in samples),
|
||||
],
|
||||
"child_peak_rss_range_mib": [
|
||||
min(x["rss_peak_mib"] for x in samples),
|
||||
max(x["rss_peak_mib"] for x in samples),
|
||||
],
|
||||
"scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned",
|
||||
}
|
||||
(a.output / "summary.json").write_text(json.dumps(stats, indent=2) + "\n")
|
||||
fig, axes = plt.subplots(1, 2, figsize=(13, 5), layout="constrained")
|
||||
axes[0].bar(
|
||||
range(1, len(rows) + 1),
|
||||
[r["publications"] for r in rows],
|
||||
color=[
|
||||
"#246b8e" if r["state"] == "ready" else "#d17d20" for r in rows
|
||||
],
|
||||
)
|
||||
axes[0].axhline(1, color="#5b6870", linestyle="--", linewidth=1)
|
||||
axes[0].set(
|
||||
xlabel="Effect observation (CSV row)",
|
||||
ylabel="Committed publications",
|
||||
title="One publication per ready version; stale v1 = 0",
|
||||
yticks=[0, 1],
|
||||
)
|
||||
axes[1].scatter(
|
||||
[x["seconds"] for x in samples],
|
||||
[x["rss_peak_mib"] for x in samples],
|
||||
color="#246b8e",
|
||||
s=35,
|
||||
alpha=0.8,
|
||||
)
|
||||
axes[1].set(
|
||||
xlabel="Child elapsed, seconds",
|
||||
ylabel="Child peak RSS, MiB",
|
||||
title=f"{len(samples)} completed child samples (synthetic320×200)",
|
||||
)
|
||||
for ax in axes:
|
||||
ax.grid(axis="y", alpha=0.2)
|
||||
fig.savefig(a.output / "pipeline-results.png", dpi=150)
|
||||
plt.close(fig)
|
||||
print(json.dumps(stats, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,256 @@
|
||||
"""Derive tables and a figure from completed observations, never simulated data."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def quantile(values, fraction):
|
||||
ordered = sorted(values)
|
||||
return (
|
||||
ordered[max(0, math.ceil(len(ordered) * fraction) - 1)]
|
||||
if ordered
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("input", type=Path)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
report = json.loads(args.input.read_text())
|
||||
assert report["complete"] and report["scope"] == "full30m"
|
||||
assert report["elapsed_seconds"] >= 1800
|
||||
with (args.input.parent / report["requests_csv"]).open() as stream:
|
||||
requests = list(csv.DictReader(stream))
|
||||
samples = report["samples"]
|
||||
origin = samples[0]["at"]
|
||||
summaries = []
|
||||
for phase in report["phases"]:
|
||||
# Labels remain unchanged while orchestration waits between holds.
|
||||
# Attribute writes by their actual start time inside the measured hold.
|
||||
rows = [
|
||||
r
|
||||
for r in requests
|
||||
if phase["start"] <= float(r["timestamp"]) < phase["end"]
|
||||
]
|
||||
successful = [
|
||||
float(r["seconds"]) for r in rows if r["status"] == "201"
|
||||
]
|
||||
elapsed = phase["end"] - phase["start"]
|
||||
summaries.append(
|
||||
{
|
||||
"phase": phase["name"],
|
||||
"seconds": round(elapsed, 3),
|
||||
"started_requests": len(rows),
|
||||
"statuses": dict(Counter(r["status"] for r in rows)),
|
||||
"successful_rps": round(len(successful) / elapsed, 3),
|
||||
"success_p50_ms": round(quantile(successful, 0.5) * 1000, 3)
|
||||
if successful
|
||||
else None,
|
||||
"success_p95_ms": round(quantile(successful, 0.95) * 1000, 3)
|
||||
if successful
|
||||
else None,
|
||||
"success_p99_ms": round(quantile(successful, 0.99) * 1000, 3)
|
||||
if successful
|
||||
else None,
|
||||
"critical_firing_names": sorted(
|
||||
n
|
||||
for n in phase["firing"]
|
||||
if n
|
||||
in {
|
||||
"ApiFastBurn",
|
||||
"ApiSlowBurn",
|
||||
"ReadyFastBurn",
|
||||
"ReadySlowBurn",
|
||||
}
|
||||
),
|
||||
"warning_firing_names": sorted(
|
||||
n
|
||||
for n in phase["firing"]
|
||||
if n
|
||||
not in {
|
||||
"ApiFastBurn",
|
||||
"ApiSlowBurn",
|
||||
"ReadyFastBurn",
|
||||
"ReadySlowBurn",
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
intervals = {}
|
||||
for name in sorted({a["name"] for s in samples for a in s["alerts"]}):
|
||||
segments, start = [], None
|
||||
for sample in samples:
|
||||
active = any(
|
||||
a["name"] == name and a["state"] == "firing"
|
||||
for a in sample["alerts"]
|
||||
)
|
||||
if active and start is None:
|
||||
start = sample["at"]
|
||||
elif not active and start is not None:
|
||||
segments.append([start, sample["at"]])
|
||||
start = None
|
||||
if start is not None:
|
||||
segments.append([start, samples[-1]["at"]])
|
||||
intervals[name] = segments
|
||||
gap = [
|
||||
b["at"] - a["at"] for a, b in zip(samples, samples[1:], strict=False)
|
||||
]
|
||||
memory = [
|
||||
s["signals"]["collector_memory"] / 1024**2
|
||||
for s in samples
|
||||
if s["signals"]["collector_memory"] is not None
|
||||
]
|
||||
summary = {
|
||||
"source_commit": report["source_commit"],
|
||||
"runtime_versions": report["runtime_versions"],
|
||||
"elapsed_seconds": report["elapsed_seconds"],
|
||||
"sample_count": len(samples),
|
||||
"sample_gap_seconds": {
|
||||
"min": min(gap),
|
||||
"median": quantile(gap, 0.5),
|
||||
"max": max(gap),
|
||||
},
|
||||
"client_quantile_method": "nearest rank of successful end-to-end writes; not histogram interpolation",
|
||||
"phase_assignment": "request start timestamp inside measured hold; orchestration gaps excluded",
|
||||
"phases": summaries,
|
||||
"incidents": report["incidents"],
|
||||
"firing_intervals_unix_seconds": intervals,
|
||||
"short_spikes": report["short_spikes"],
|
||||
"collector_rss_mib": {"min": min(memory), "max": max(memory)},
|
||||
"collector_max_queued_packets_sum": max(
|
||||
s["signals"]["collector_queue"] or 0 for s in samples
|
||||
),
|
||||
"collector_queue_capacity_sum": max(
|
||||
s["signals"]["collector_queue_capacity"] or 0 for s in samples
|
||||
),
|
||||
"backend_outage": report["backend_outage"],
|
||||
"preservation": report["preservation"],
|
||||
"final_30m_sli": report["final_30m_sli"],
|
||||
}
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
(args.output / "summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
with (args.output / "phases.csv").open("w") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(summaries[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(summaries)
|
||||
x = [(s["at"] - origin) / 60 for s in samples]
|
||||
fig, axes = plt.subplots(
|
||||
4, 1, figsize=(12, 12), sharex=True, layout="constrained"
|
||||
)
|
||||
for axis in axes:
|
||||
axis.grid(alpha=0.2)
|
||||
for phase in report["phases"]:
|
||||
if (
|
||||
phase["name"].endswith("_sustained")
|
||||
or phase["name"] == "backend_unavailable"
|
||||
):
|
||||
color = (
|
||||
"#cf704e"
|
||||
if phase["name"].startswith("api")
|
||||
else "#8b79b5"
|
||||
if phase["name"].startswith("ready")
|
||||
else "#85919d"
|
||||
)
|
||||
axis.axvspan(
|
||||
(phase["start"] - origin) / 60,
|
||||
(phase["end"] - origin) / 60,
|
||||
color=color,
|
||||
alpha=0.15,
|
||||
)
|
||||
axes[0].plot(
|
||||
x,
|
||||
[(s["signals"]["api_error_ratio_1m"] or 0) * 100 for s in samples],
|
||||
color="#b04a32",
|
||||
label="API 5xx ratio, rolling 1 min",
|
||||
)
|
||||
axes[0].axhline(1, color="black", ls=":", label="1% error budget")
|
||||
axes[0].set_ylabel("5xx, %")
|
||||
axes[0].legend(loc="upper right")
|
||||
axes[0].set_title(
|
||||
"Actual 30-minute SLO experiment: DB delay, stopped worker, unavailable telemetry stores"
|
||||
)
|
||||
axes[1].plot(
|
||||
x,
|
||||
[s["signals"]["queue_depth"] for s in samples],
|
||||
color="#386a9b",
|
||||
label="Queued jobs",
|
||||
)
|
||||
axes[1].set_ylabel("Jobs")
|
||||
other = axes[1].twinx()
|
||||
other.plot(
|
||||
x,
|
||||
[s["signals"]["oldest_job_age"] for s in samples],
|
||||
color="#8b579d",
|
||||
label="Oldest unfinished job",
|
||||
)
|
||||
other.axhline(20, ls=":", color="#8b579d")
|
||||
other.set_ylabel("Age, seconds")
|
||||
axes[1].legend(loc="upper left")
|
||||
other.legend(loc="upper right")
|
||||
alert_names = [
|
||||
"ApiFastBurn",
|
||||
"ApiSlowBurn",
|
||||
"ReadyFastBurn",
|
||||
"ReadySlowBurn",
|
||||
]
|
||||
for index, name in enumerate(alert_names):
|
||||
axes[2].broken_barh(
|
||||
[
|
||||
((start - origin) / 60, (end - start) / 60)
|
||||
for start, end in intervals.get(name, [])
|
||||
],
|
||||
(index - 0.3, 0.6),
|
||||
facecolors="#bd523c",
|
||||
)
|
||||
axes[2].set_yticks(range(4), alert_names)
|
||||
axes[2].set_ylim(-0.6, 3.6)
|
||||
axes[2].set_ylabel("Critical alert firing")
|
||||
axes[3].plot(
|
||||
x,
|
||||
[
|
||||
s["signals"]["collector_memory"] / 1024**2
|
||||
if s["signals"]["collector_memory"] is not None
|
||||
else None
|
||||
for s in samples
|
||||
],
|
||||
color="#347465",
|
||||
label="Collector RSS",
|
||||
)
|
||||
axes[3].axhline(
|
||||
report["container_limits"]["otel"]["mem_limit"] / 1024**2,
|
||||
ls=":",
|
||||
color="black",
|
||||
label="Container memory limit",
|
||||
)
|
||||
axes[3].set_ylabel("MiB")
|
||||
axes[3].set_xlabel("Minutes from first observation")
|
||||
other = axes[3].twinx()
|
||||
other.plot(
|
||||
x,
|
||||
[s["signals"]["collector_queue"] for s in samples],
|
||||
color="#9c6e2e",
|
||||
label="Queued telemetry packets, sum",
|
||||
)
|
||||
other.set_ylabel("Packets")
|
||||
axes[3].legend(loc="upper left")
|
||||
other.legend(loc="upper right")
|
||||
fig.savefig(args.output / "slo-timeline.png", dpi=150)
|
||||
plt.close(fig)
|
||||
print("PASS: measured phase statistics, alert intervals and timeline")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def create(path, value):
|
||||
try:
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
||||
except FileExistsError:
|
||||
return
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
stream.write(value + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
directory = ROOT / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
pg = directory / "postgres_password"
|
||||
cache = directory / "cache_password"
|
||||
create(pg, secrets.token_urlsafe(32))
|
||||
create(cache, secrets.token_urlsafe(32))
|
||||
create(
|
||||
directory / "database_url",
|
||||
"postgresql://messenger:"
|
||||
+ pg.read_text().strip()
|
||||
+ "@postgres:5432/messenger",
|
||||
)
|
||||
create(
|
||||
directory / "cache_url",
|
||||
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
|
||||
)
|
||||
create(
|
||||
directory / "valkey.conf",
|
||||
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
|
||||
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
|
||||
+ cache.read_text().strip(),
|
||||
)
|
||||
create(directory / "s3_root_user", secrets.token_hex(10))
|
||||
create(directory / "s3_root_password", secrets.token_urlsafe(32))
|
||||
create(directory / "s3_access_key", secrets.token_hex(10))
|
||||
create(directory / "s3_secret_key", secrets.token_urlsafe(32))
|
||||
create(directory / "grafana_password", secrets.token_urlsafe(32))
|
||||
create(directory / "broker_password", secrets.token_urlsafe(32))
|
||||
create(
|
||||
directory / "broker_url",
|
||||
"redis://:"
|
||||
+ (directory / "broker_password").read_text().strip()
|
||||
+ "@broker:6379/0",
|
||||
)
|
||||
create(
|
||||
directory / "broker.conf",
|
||||
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass "
|
||||
+ (directory / "broker_password").read_text().strip(),
|
||||
)
|
||||
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
|
||||
print("Local configuration ready; existing files preserved")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Явные операторские действия только над API текущего Compose-проекта."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def containers():
|
||||
ids = command("docker", "compose", "ps", "-q", "api").split()
|
||||
return json.loads(command("docker", "inspect", *ids)) if ids else []
|
||||
|
||||
|
||||
def alias(cid):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=2); "
|
||||
"print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
return command("docker", "exec", cid, "python", "-c", code)
|
||||
|
||||
|
||||
def wait_count(count):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
data = containers()
|
||||
if len(data) == count and all(
|
||||
c["State"].get("Health", {}).get("Status") == "healthy"
|
||||
for c in data
|
||||
):
|
||||
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
|
||||
time.sleep(3)
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError("Replica readiness")
|
||||
|
||||
|
||||
def main():
|
||||
action = sys.argv[1]
|
||||
data = containers()
|
||||
if action == "stop":
|
||||
target = os.environ["TARGET_INSTANCE_ID"]
|
||||
matches = [c for c in data if alias(c["Id"]) == target]
|
||||
if len(matches) != 1 or len(data) < 2:
|
||||
raise RuntimeError("Target must identify one running replica")
|
||||
command("docker", "stop", "--time", "10", matches[0]["Id"])
|
||||
print("Selected API stopped")
|
||||
elif action == "rolling":
|
||||
count = len(data)
|
||||
if count < 2:
|
||||
raise RuntimeError("Need at least two replicas")
|
||||
# Сначала добавляется готовая реплика, затем удаляется одна старая.
|
||||
for old in data:
|
||||
command(
|
||||
"docker",
|
||||
"compose",
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--no-recreate",
|
||||
"--scale",
|
||||
f"api={count + 1}",
|
||||
"api",
|
||||
)
|
||||
wait_count(count + 1)
|
||||
command("docker", "stop", "--time", "10", old["Id"])
|
||||
command("docker", "rm", old["Id"])
|
||||
wait_count(count)
|
||||
print("Sequential replacement completed")
|
||||
else:
|
||||
raise ValueError("Unknown action")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Read-only queries through an internal probe, without exposing admin ports."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from scripts.stand import compose
|
||||
|
||||
ENDPOINTS = {
|
||||
"prometheus": "http://prometheus:9090",
|
||||
"loki": "http://loki:3100",
|
||||
"tempo": "http://tempo:3200",
|
||||
"otel": "http://otel:13133",
|
||||
}
|
||||
|
||||
|
||||
def fetch(service, path, *, params=None, json_result=True):
|
||||
url = ENDPOINTS[service] + path
|
||||
if params:
|
||||
url += "?" + urlencode(params, doseq=True)
|
||||
code = (
|
||||
"import urllib.request; "
|
||||
f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8); "
|
||||
"print(r.read().decode())"
|
||||
)
|
||||
text = compose("exec", "-T", "observer", "python", "-c", code)
|
||||
return json.loads(text) if json_result else text
|
||||
|
||||
|
||||
def query(expression, *, at=None):
|
||||
params = {"query": expression}
|
||||
if at is not None:
|
||||
params["time"] = str(at)
|
||||
result = fetch("prometheus", "/api/v1/query", params=params)
|
||||
assert result["status"] == "success"
|
||||
return result["data"]["result"]
|
||||
|
||||
|
||||
def scalar(expression):
|
||||
result = query(expression)
|
||||
return float(result[0]["value"][1]) if result else 0.0
|
||||
|
||||
|
||||
def queries(expressions):
|
||||
urls = {
|
||||
name: ENDPOINTS["prometheus"]
|
||||
+ "/api/v1/query?"
|
||||
+ urlencode({"query": expression})
|
||||
for name, expression in expressions.items()
|
||||
}
|
||||
code = (
|
||||
"import json,urllib.request\n"
|
||||
f"urls={urls!r}\n"
|
||||
"result={}\n"
|
||||
"for name,url in urls.items():\n"
|
||||
" with urllib.request.urlopen(url,timeout=8) as r: result[name]=json.load(r)['data']['result']\n"
|
||||
"print(json.dumps(result))"
|
||||
)
|
||||
return json.loads(compose("exec", "-T", "observer", "python", "-c", code))
|
||||
|
||||
|
||||
def logs(trace_id, *, start=None):
|
||||
return fetch(
|
||||
"loki",
|
||||
"/loki/api/v1/query_range",
|
||||
params={
|
||||
"query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="'
|
||||
+ trace_id
|
||||
+ '"',
|
||||
"start": str(int((start or time.time() - 3600) * 1e9)),
|
||||
"end": str(time.time_ns()),
|
||||
"limit": 500,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def normalize_id(value):
|
||||
if not value:
|
||||
return ""
|
||||
if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value):
|
||||
return value
|
||||
return base64.b64decode(value).hex()
|
||||
|
||||
|
||||
def spans(trace_id):
|
||||
result = fetch("tempo", "/api/traces/" + trace_id)
|
||||
flattened = []
|
||||
for batch in result.get("batches", result.get("resourceSpans", [])):
|
||||
resource = {
|
||||
a["key"]: next(iter(a["value"].values()))
|
||||
for a in batch.get("resource", {}).get("attributes", [])
|
||||
}
|
||||
for scope in batch.get("scopeSpans", []):
|
||||
for item in scope.get("spans", []):
|
||||
flattened.append(
|
||||
{
|
||||
"service": resource.get("service.name"),
|
||||
"version": resource.get("service.version"),
|
||||
"instance": resource.get("service.instance.id"),
|
||||
"name": item["name"],
|
||||
"kind": item.get("kind"),
|
||||
"trace_id": normalize_id(item.get("traceId")),
|
||||
"span_id": normalize_id(item.get("spanId")),
|
||||
"parent_span_id": normalize_id(
|
||||
item.get("parentSpanId")
|
||||
),
|
||||
"duration_seconds": (
|
||||
int(item["endTimeUnixNano"])
|
||||
- int(item["startTimeUnixNano"])
|
||||
)
|
||||
/ 1e9,
|
||||
"status": item.get("status", {}),
|
||||
"attributes": {
|
||||
a["key"]: next(iter(a["value"].values()))
|
||||
for a in item.get("attributes", [])
|
||||
},
|
||||
}
|
||||
)
|
||||
return flattened
|
||||
|
||||
|
||||
def wait(condition, *, timeout=90, interval=1):
|
||||
end = time.monotonic() + timeout
|
||||
last_error = None
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
result = condition()
|
||||
if result:
|
||||
return result
|
||||
except (RuntimeError, ValueError, KeyError) as exc:
|
||||
last_error = type(exc).__name__
|
||||
time.sleep(interval)
|
||||
raise TimeoutError("Telemetry observation not reached: " + str(last_error))
|
||||
@@ -0,0 +1,633 @@
|
||||
"""Thirty-minute controlled SLO study with real DB/worker/backend failures."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
||||
from scripts.observe import fetch, logs, queries, spans, wait
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
||||
|
||||
ERRORS = 'haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}'
|
||||
RESPONSES = 'haproxy_backend_http_responses_total{proxy="messenger"}'
|
||||
SIGNALS = {
|
||||
"api_error_ratio_1m": f"sum(rate({ERRORS}[1m])) / clamp_min(sum(rate({RESPONSES}[1m])),0.001)",
|
||||
"queue_depth": "max(messenger_queue_depth)",
|
||||
"oldest_job_age": "max(messenger_oldest_job_age_seconds)",
|
||||
"worker_busy": "sum(messenger_worker_busy)",
|
||||
"worker_attempts": "sum(messenger_job_attempts_total)",
|
||||
"pool_used": 'sum(messenger_db_pool_used{service_name="messenger-api"})',
|
||||
"pool_waiting": 'sum(messenger_db_pool_waiting{service_name="messenger-api"})',
|
||||
"http_p95": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"slo_good": 'sum(messenger_slo_decisions_total{outcome="good"})',
|
||||
"slo_bad": 'sum(messenger_slo_decisions_total{outcome="bad"})',
|
||||
"collector_received_spans": "sum(otelcol_receiver_accepted_spans)",
|
||||
"collector_sent_spans": "sum(otelcol_exporter_sent_spans)",
|
||||
"collector_received_logs": "sum(otelcol_receiver_accepted_log_records)",
|
||||
"collector_sent_logs": "sum(otelcol_exporter_sent_log_records)",
|
||||
"collector_failed_spans": "sum(otelcol_exporter_send_failed_spans)",
|
||||
"collector_queue": "sum(otelcol_exporter_queue_size)",
|
||||
"collector_queue_capacity": "sum(otelcol_exporter_queue_capacity)",
|
||||
"collector_memory": 'otelcol_process_memory_rss{job="collector"}',
|
||||
"app_memory": "sum(messenger_process_memory_bytes)",
|
||||
}
|
||||
BURN_ALERTS = {"ApiFastBurn", "ApiSlowBurn", "ReadyFastBurn", "ReadySlowBurn"}
|
||||
|
||||
|
||||
def snapshot(phase):
|
||||
observed = queries(SIGNALS)
|
||||
values = {}
|
||||
for key, rows in observed.items():
|
||||
value = float(rows[0]["value"][1]) if rows else None
|
||||
values[key] = (
|
||||
value if value is not None and math.isfinite(value) else None
|
||||
)
|
||||
alerts = fetch("prometheus", "/api/v1/alerts")["data"]["alerts"]
|
||||
return {
|
||||
"at": time.time(),
|
||||
"phase": phase,
|
||||
"signals": values,
|
||||
"alerts": [
|
||||
{
|
||||
"name": a["labels"]["alertname"],
|
||||
"state": a["state"],
|
||||
"active_at": a.get("activeAt"),
|
||||
}
|
||||
for a in alerts
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def firing(samples):
|
||||
return {
|
||||
a["name"]
|
||||
for sample in samples
|
||||
for a in sample["alerts"]
|
||||
if a["state"] == "firing"
|
||||
}
|
||||
|
||||
|
||||
class Workload:
|
||||
def __init__(self, client, headers, chat):
|
||||
self.client, self.headers, self.chat = client, headers, chat
|
||||
self.stop = threading.Event()
|
||||
self.phase = "warmup"
|
||||
self.requests = []
|
||||
self.uploads = []
|
||||
self.lock = threading.Lock()
|
||||
self.slots = threading.BoundedSemaphore(16)
|
||||
self.pool = ThreadPoolExecutor(max_workers=16)
|
||||
image = io.BytesIO()
|
||||
Image.new("RGB", (320, 200), (39, 102, 157)).save(image, "PNG")
|
||||
self.raw = image.getvalue()
|
||||
self.threads = [
|
||||
threading.Thread(target=self.schedule_messages),
|
||||
threading.Thread(target=self.schedule_uploads),
|
||||
]
|
||||
|
||||
def write(self):
|
||||
started, phase, trace_id = time.time(), self.phase, uuid4().hex
|
||||
status, identity = 0, ""
|
||||
try:
|
||||
response = self.client.post(
|
||||
f"/api/v1/chats/{self.chat}/messages",
|
||||
headers={
|
||||
**self.headers,
|
||||
"Idempotency-Key": uuid4().hex,
|
||||
"traceparent": "00-"
|
||||
+ trace_id
|
||||
+ "-"
|
||||
+ uuid4().hex[:16]
|
||||
+ "-01",
|
||||
},
|
||||
json={"text": "Synthetic SLO message " + uuid4().hex},
|
||||
)
|
||||
status = response.status_code
|
||||
if status == 201:
|
||||
identity = response.json()["id"]
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
finally:
|
||||
with self.lock:
|
||||
self.requests.append(
|
||||
[
|
||||
started,
|
||||
phase,
|
||||
status,
|
||||
time.time() - started,
|
||||
trace_id,
|
||||
identity,
|
||||
]
|
||||
)
|
||||
self.slots.release()
|
||||
|
||||
def schedule_messages(self):
|
||||
next_at = time.monotonic()
|
||||
while not self.stop.is_set():
|
||||
if self.slots.acquire(blocking=False):
|
||||
self.pool.submit(self.write)
|
||||
else:
|
||||
with self.lock:
|
||||
self.requests.append(
|
||||
[time.time(), self.phase, -1, 0, "", ""]
|
||||
)
|
||||
next_at += 0.25
|
||||
self.stop.wait(max(0, next_at - time.monotonic()))
|
||||
|
||||
def schedule_uploads(self):
|
||||
while not self.stop.is_set():
|
||||
started, phase, trace_id = time.time(), self.phase, uuid4().hex
|
||||
status, identity = 0, ""
|
||||
try:
|
||||
response = self.client.post(
|
||||
f"/api/v1/chats/{self.chat}/attachments",
|
||||
params={"filename": "synthetic-slo.png"},
|
||||
content=self.raw,
|
||||
headers={
|
||||
**self.headers,
|
||||
"Content-Type": "image/png",
|
||||
"traceparent": "00-"
|
||||
+ trace_id
|
||||
+ "-"
|
||||
+ uuid4().hex[:16]
|
||||
+ "-01",
|
||||
},
|
||||
)
|
||||
status = response.status_code
|
||||
if status == 202:
|
||||
identity = response.json()["id"]
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
with self.lock:
|
||||
self.uploads.append(
|
||||
[started, phase, status, identity, trace_id]
|
||||
)
|
||||
self.stop.wait(max(0, 5 - (time.time() - started)))
|
||||
|
||||
def start(self):
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
|
||||
def close(self):
|
||||
self.stop.set()
|
||||
for thread in self.threads:
|
||||
thread.join(20)
|
||||
self.pool.shutdown(wait=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/slo-research-first.json")
|
||||
parser.add_argument("--minimum-seconds", type=int, default=1800)
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
assert not output.exists(), "Choose a new output path"
|
||||
report = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"scope": "full30m"
|
||||
if args.minimum_seconds >= 1800
|
||||
else "short-development",
|
||||
"profile": {
|
||||
"messages_per_second": 4,
|
||||
"max_inflight": 16,
|
||||
"upload_every_seconds": 5,
|
||||
"image_pixels": [320, 200],
|
||||
},
|
||||
"complete": False,
|
||||
"samples": [],
|
||||
"phases": [],
|
||||
}
|
||||
study_started = time.monotonic()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
load = None
|
||||
trigger = "lab6_slo_delay_" + uuid4().hex[:8]
|
||||
with httpx.Client(
|
||||
base_url=BASE,
|
||||
timeout=8,
|
||||
trust_env=False,
|
||||
limits=httpx.Limits(max_connections=24, max_keepalive_connections=24),
|
||||
) as client:
|
||||
ready(timeout=90)
|
||||
config = json.loads(compose("config", "--format", "json"))
|
||||
report["container_limits"] = {
|
||||
name: {
|
||||
k: int(service[k])
|
||||
if k in {"mem_limit", "memswap_limit", "scale"}
|
||||
else service[k]
|
||||
for k in ["cpus", "mem_limit", "memswap_limit", "scale"]
|
||||
if k in service
|
||||
}
|
||||
for name, service in config["services"].items()
|
||||
}
|
||||
report["docker_vm"] = json.loads(
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"info",
|
||||
"--format",
|
||||
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}',
|
||||
]
|
||||
)
|
||||
)
|
||||
report["runtime_versions"] = sorted(
|
||||
{
|
||||
row["metric"].get("service_version", "unknown")
|
||||
for row in queries(
|
||||
{"versions": "messenger_process_memory_bytes"}
|
||||
)["versions"]
|
||||
}
|
||||
)
|
||||
user, _, session = actor(client)
|
||||
headers = auth(session)
|
||||
response = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers,
|
||||
json={
|
||||
"title": "Synthetic SLO experiment",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 201
|
||||
chat = response.json()["id"]
|
||||
load = Workload(client, headers, chat)
|
||||
workload = load
|
||||
|
||||
def save():
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
|
||||
def hold(name, seconds, *, until_clear=None, max_seconds=None):
|
||||
load.phase = name
|
||||
phase = {
|
||||
"name": name,
|
||||
"start": time.time(),
|
||||
"planned_seconds": seconds,
|
||||
"maximum_seconds": max_seconds or seconds,
|
||||
}
|
||||
report["phases"].append(phase)
|
||||
minimum_end = time.monotonic() + seconds
|
||||
deadline = time.monotonic() + (max_seconds or seconds)
|
||||
start_index = len(report["samples"])
|
||||
while time.monotonic() < deadline:
|
||||
sample = snapshot(name)
|
||||
report["samples"].append(sample)
|
||||
save()
|
||||
if (
|
||||
until_clear is not None
|
||||
and time.monotonic() >= minimum_end
|
||||
and not until_clear.intersection(firing([sample]))
|
||||
):
|
||||
break
|
||||
time.sleep(min(5, max(0, deadline - time.monotonic())))
|
||||
phase["end"] = time.time()
|
||||
phase["firing"] = sorted(firing(report["samples"][start_index:]))
|
||||
return phase
|
||||
|
||||
def db_delay(on):
|
||||
if on:
|
||||
sql(
|
||||
f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='{chat}'::uuid THEN PERFORM pg_sleep(3); END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {trigger}()"
|
||||
)
|
||||
else:
|
||||
sql(
|
||||
f"DROP TRIGGER IF EXISTS {trigger} ON messages; DROP FUNCTION IF EXISTS {trigger}()"
|
||||
)
|
||||
|
||||
try:
|
||||
load.start()
|
||||
warm = hold("warmup", 300)
|
||||
settled = [
|
||||
s for s in report["samples"] if s["at"] >= warm["end"] - 60
|
||||
]
|
||||
assert not BURN_ALERTS.intersection(firing(settled)), (
|
||||
"Baseline is not clean"
|
||||
)
|
||||
assert (
|
||||
report["samples"][-1]["signals"]["collector_memory"]
|
||||
is not None
|
||||
)
|
||||
report["baseline"] = snapshot("baseline")
|
||||
for signal in ["api", "ready"]:
|
||||
if signal == "api":
|
||||
activate, restore = (
|
||||
lambda: db_delay(True),
|
||||
lambda: db_delay(False),
|
||||
)
|
||||
else:
|
||||
|
||||
def activate():
|
||||
return compose("stop", "--timeout", "1", "worker")
|
||||
|
||||
def restore():
|
||||
return compose(
|
||||
"up", "-d", "--no-deps", "--no-recreate", "worker"
|
||||
)
|
||||
|
||||
brief_started = time.time()
|
||||
activate()
|
||||
brief = hold(signal + "_brief", 5)
|
||||
restore()
|
||||
settle = hold(
|
||||
signal + "_brief_recovery", 120 if signal == "api" else 60
|
||||
)
|
||||
assert not BURN_ALERTS.intersection(
|
||||
brief["firing"] + settle["firing"]
|
||||
), "Brief incident triggered paging"
|
||||
report.setdefault("short_spikes", []).append(
|
||||
{
|
||||
"signal": signal,
|
||||
"action_started": brief_started,
|
||||
"paging_events": 0,
|
||||
}
|
||||
)
|
||||
action_started = time.time()
|
||||
activate()
|
||||
action_finished = time.time()
|
||||
degraded = hold(
|
||||
signal + "_sustained", 180 if signal == "api" else 150
|
||||
)
|
||||
restore_started = time.time()
|
||||
restore()
|
||||
restore_finished = time.time()
|
||||
expected = (
|
||||
{"ApiFastBurn", "ApiSlowBurn", "ApiHighErrorRate"}
|
||||
if signal == "api"
|
||||
else {"ReadyFastBurn", "ReadySlowBurn", "QueueLagGrowing"}
|
||||
)
|
||||
recovered = hold(
|
||||
signal + "_recovery",
|
||||
150 if signal == "api" else 240,
|
||||
until_clear=expected,
|
||||
max_seconds=600,
|
||||
)
|
||||
assert expected <= set(degraded["firing"]), (
|
||||
signal,
|
||||
degraded["firing"],
|
||||
)
|
||||
last = report["samples"][-1]
|
||||
assert not expected.intersection(firing([last])), (
|
||||
"Incident alerts did not resolve"
|
||||
)
|
||||
points = [
|
||||
s
|
||||
for s in report["samples"]
|
||||
if action_finished <= s["at"] <= recovered["end"]
|
||||
]
|
||||
timings = {}
|
||||
for name in sorted(expected):
|
||||
first = next(
|
||||
s["at"] for s in points if name in firing([s])
|
||||
)
|
||||
resolved = next(
|
||||
s["at"]
|
||||
for s in points
|
||||
if s["at"] >= restore_finished
|
||||
and name not in firing([s])
|
||||
)
|
||||
timings[name] = {
|
||||
"detect_seconds": round(first - action_finished, 3),
|
||||
"resolve_seconds": round(
|
||||
resolved - restore_finished, 3
|
||||
),
|
||||
}
|
||||
report.setdefault("incidents", []).append(
|
||||
{
|
||||
"signal": signal,
|
||||
"action_started": action_started,
|
||||
"action_finished": action_finished,
|
||||
"restore_started": restore_started,
|
||||
"restore_finished": restore_finished,
|
||||
"timings": timings,
|
||||
}
|
||||
)
|
||||
if signal == "api":
|
||||
candidates = [
|
||||
r
|
||||
for r in load.requests
|
||||
if action_finished <= r[0] <= restore_started
|
||||
and r[2] >= 500
|
||||
]
|
||||
assert candidates
|
||||
items = wait(
|
||||
lambda identity=candidates[0][4]: spans(identity)
|
||||
)
|
||||
assert any(
|
||||
s["name"] == "db.postgresql"
|
||||
and s["duration_seconds"] >= 2
|
||||
and s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
|
||||
for s in items
|
||||
)
|
||||
report["database_diagnostic_trace"] = items
|
||||
else:
|
||||
assert (
|
||||
max(
|
||||
s["signals"]["oldest_job_age"] or 0 for s in points
|
||||
)
|
||||
> 20
|
||||
)
|
||||
assert (
|
||||
max(s["signals"]["queue_depth"] or 0 for s in points)
|
||||
> 0
|
||||
)
|
||||
accepted = [
|
||||
r
|
||||
for r in load.uploads
|
||||
if action_finished <= r[0] <= restore_started
|
||||
and r[2] == 202
|
||||
]
|
||||
assert accepted
|
||||
report["worker_recovered_trace"] = wait(
|
||||
lambda identity=accepted[0][4]: spans(identity)
|
||||
)
|
||||
report["worker_recovered_attachment"] = accepted[0][3]
|
||||
outage_before = snapshot("backend_before")
|
||||
outage_started = time.time()
|
||||
compose("stop", "--timeout", "1", "loki", "tempo")
|
||||
outage = hold("backend_unavailable", 90)
|
||||
points = [
|
||||
s
|
||||
for s in report["samples"]
|
||||
if s["phase"] == "backend_unavailable"
|
||||
]
|
||||
assert max(
|
||||
s["signals"]["collector_failed_spans"] or 0 for s in points
|
||||
) > (outage_before["signals"]["collector_failed_spans"] or 0)
|
||||
assert all(
|
||||
s["signals"]["collector_queue"] is not None
|
||||
and s["signals"]["collector_queue_capacity"] is not None
|
||||
and s["signals"]["collector_memory"] is not None
|
||||
for s in points
|
||||
)
|
||||
assert all(
|
||||
(s["signals"]["collector_queue"] or 0)
|
||||
<= (s["signals"]["collector_queue_capacity"] or 0)
|
||||
for s in points
|
||||
)
|
||||
assert (
|
||||
max(s["signals"]["collector_memory"] for s in points)
|
||||
< report["container_limits"]["otel"]["mem_limit"]
|
||||
)
|
||||
assert not BURN_ALERTS.intersection(outage["firing"])
|
||||
during = [
|
||||
r for r in load.requests if r[1] == "backend_unavailable"
|
||||
]
|
||||
assert during and all(r[2] == 201 for r in during)
|
||||
before_ready = sql(
|
||||
"SELECT count(*) FROM attachments WHERE state='ready'"
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "loki", "tempo")
|
||||
wait(lambda: fetch("loki", "/ready", json_result=False))
|
||||
wait(lambda: fetch("tempo", "/ready", json_result=False))
|
||||
hold("backend_recovery", 60)
|
||||
restored = next(
|
||||
r
|
||||
for r in reversed(load.uploads)
|
||||
if r[1] == "backend_recovery" and r[2] == 202
|
||||
)
|
||||
wait(
|
||||
lambda: any(
|
||||
s["name"] == "attachment.process"
|
||||
for s in spans(restored[4])
|
||||
)
|
||||
)
|
||||
wait(lambda: logs(restored[4]).get("data", {}).get("result"))
|
||||
report["backend_outage"] = {
|
||||
"started": outage_started,
|
||||
"finished": outage["end"],
|
||||
"before": outage_before,
|
||||
"after": snapshot("backend_after"),
|
||||
"ready_count_before_backend_recovery": int(before_ready),
|
||||
"successful_writes_during_outage": len(during),
|
||||
"restored_trace_id": restored[4],
|
||||
"queue_bounded": True,
|
||||
"collector_rss_below_limit": True,
|
||||
"send_failure_counter_counts_attempts_not_unique_loss": True,
|
||||
}
|
||||
remaining = args.minimum_seconds - (
|
||||
time.monotonic() - study_started
|
||||
)
|
||||
if remaining > 0:
|
||||
hold("steady_after_recovery", remaining)
|
||||
load.close()
|
||||
load = None
|
||||
# Read through the public API: a successful SQL insert alone is not
|
||||
# proof that every acknowledged message remains accessible.
|
||||
seen, cursor = set(), None
|
||||
while True:
|
||||
page = client.get(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers=headers,
|
||||
params={
|
||||
"limit": 100,
|
||||
**({"cursor": cursor} if cursor else {}),
|
||||
},
|
||||
)
|
||||
page.raise_for_status()
|
||||
data = page.json()
|
||||
ids = [row["id"] for row in data["items"]]
|
||||
assert len(ids) == len(set(ids)) and not seen.intersection(ids)
|
||||
seen.update(ids)
|
||||
cursor = data.get("next_cursor")
|
||||
if not cursor:
|
||||
break
|
||||
expected_messages = {
|
||||
r[5] for r in workload.requests if r[2] == 201
|
||||
}
|
||||
assert seen == expected_messages
|
||||
accepted_uploads = [r for r in workload.uploads if r[2] == 202]
|
||||
for row in accepted_uploads:
|
||||
|
||||
def completed(identity=row[3]):
|
||||
response = client.get(
|
||||
f"/api/v1/attachments/{identity}", headers=headers
|
||||
)
|
||||
return (
|
||||
response.status_code == 200
|
||||
and response.json()["status"] == "ready"
|
||||
)
|
||||
|
||||
wait(completed, timeout=120)
|
||||
for row in accepted_uploads[
|
||||
:: max(1, len(accepted_uploads) // 10)
|
||||
]:
|
||||
response = client.get(
|
||||
f"/api/v1/attachments/{row[3]}/content", headers=headers
|
||||
)
|
||||
assert (
|
||||
response.status_code == 200
|
||||
and response.content == workload.raw
|
||||
)
|
||||
report["preservation"] = {
|
||||
"acknowledged_messages": len(expected_messages),
|
||||
"accessible_messages": len(seen),
|
||||
"accepted_uploads": len(accepted_uploads),
|
||||
"all_accepted_uploads_ready": True,
|
||||
"sampled_originals_match": True,
|
||||
}
|
||||
report["rules"] = fetch("prometheus", "/api/v1/rules")
|
||||
report["final_30m_sli"] = queries(
|
||||
{
|
||||
"api_good_ratio": f"1 - sum(increase({ERRORS}[30m])) / clamp_min(sum(increase({RESPONSES}[30m])),1)",
|
||||
"ready_good_ratio": 'sum(increase(messenger_slo_decisions_total{outcome="good"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)',
|
||||
}
|
||||
)
|
||||
report["complete"] = True
|
||||
finally:
|
||||
try:
|
||||
db_delay(False)
|
||||
finally:
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--no-recreate",
|
||||
"worker",
|
||||
"loki",
|
||||
"tempo",
|
||||
)
|
||||
if load is not None:
|
||||
load.close()
|
||||
report["elapsed_seconds"] = round(
|
||||
time.monotonic() - study_started, 3
|
||||
)
|
||||
request_path = output.with_name(output.stem + "-requests.csv")
|
||||
with request_path.open("w") as stream:
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(
|
||||
[
|
||||
"timestamp",
|
||||
"phase",
|
||||
"status",
|
||||
"seconds",
|
||||
"trace_id",
|
||||
"message_id",
|
||||
]
|
||||
)
|
||||
writer.writerows(workload.requests)
|
||||
report["requests_csv"] = request_path.name
|
||||
report["request_statuses"] = dict(
|
||||
Counter(str(r[2]) for r in workload.requests)
|
||||
)
|
||||
report["uploads"] = workload.uploads
|
||||
save()
|
||||
print(
|
||||
"PASS: short/sustained SLO incidents, detection/resolution and controlled false paging"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
docker compose restart api postgres cache s3 broker relay worker lb
|
||||
docker compose up -d --no-build --no-recreate --wait
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 scripts/cluster.py rolling
|
||||
@@ -0,0 +1,137 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8086")
|
||||
|
||||
|
||||
def command(args, *, input=None):
|
||||
result = subprocess.run(
|
||||
args, cwd=ROOT, input=input, capture_output=True, text=True
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError(
|
||||
"Command failed: "
|
||||
+ args[0]
|
||||
+ "; "
|
||||
+ result.stderr.replace(str(ROOT), "<repository>")
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return command(["docker", "compose", *args])
|
||||
|
||||
|
||||
def sql(query, *, database="messenger"):
|
||||
return compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
database,
|
||||
"-A",
|
||||
"-t",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-c",
|
||||
query,
|
||||
)
|
||||
|
||||
|
||||
def cache(command_name, *args):
|
||||
script = (
|
||||
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
|
||||
'exec valkey-cli "$@"'
|
||||
)
|
||||
return compose(
|
||||
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
|
||||
)
|
||||
|
||||
|
||||
def ready(base=BASE, *, timeout=30):
|
||||
end = time.monotonic() + timeout
|
||||
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
if client.get("/health/ready").status_code == 200:
|
||||
return
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise TimeoutError("API readiness")
|
||||
|
||||
|
||||
def actor(client):
|
||||
username = "verify_" + uuid4().hex[:20]
|
||||
password = "Synthetic-" + uuid4().hex + ":🌍"
|
||||
result = client.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"username": username,
|
||||
"display_name": "Synthetic",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
assert result.status_code == 201
|
||||
user = result.json()
|
||||
session = client.post(
|
||||
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
|
||||
)
|
||||
assert session.status_code == 201
|
||||
return user, password, session.json()
|
||||
|
||||
|
||||
def auth(session):
|
||||
return {"Authorization": "Bearer " + session["token"]}
|
||||
|
||||
|
||||
def extra_api(*, ttl=3600, pool_size=1):
|
||||
name = "mtusi-lab6-test-" + uuid4().hex[:10]
|
||||
compose(
|
||||
"run",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
name,
|
||||
"-e",
|
||||
"SESSION_TTL_SECONDS=" + str(ttl),
|
||||
"-e",
|
||||
"DB_POOL_SIZE=" + str(pool_size),
|
||||
"-p",
|
||||
"127.0.0.1::8080",
|
||||
"api",
|
||||
)
|
||||
try:
|
||||
info = json.loads(command(["docker", "inspect", name]))[0]
|
||||
base = (
|
||||
"http://127.0.0.1:"
|
||||
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
|
||||
)
|
||||
ready(base)
|
||||
return name, base
|
||||
except Exception:
|
||||
command(["docker", "rm", "-f", name])
|
||||
raise
|
||||
|
||||
|
||||
def instances():
|
||||
data = {}
|
||||
for role in ["api", "postgres", "cache"]:
|
||||
cid = compose("ps", "-q", role)
|
||||
info = json.loads(command(["docker", "inspect", cid]))[0]
|
||||
data[role] = {
|
||||
"container": cid[:12],
|
||||
"pid": info["State"]["Pid"],
|
||||
"started_at": info["State"]["StartedAt"],
|
||||
}
|
||||
return data
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 scripts/cluster.py stop
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
docker compose stop --timeout 1 worker
|
||||
@@ -0,0 +1,338 @@
|
||||
import argparse
|
||||
import atexit
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROGRESS = {}
|
||||
|
||||
|
||||
def save_incomplete():
|
||||
receipt = PROGRESS.get("receipt")
|
||||
if receipt and receipt.get("result") != "PASS":
|
||||
target = PROGRESS["output"].with_name(
|
||||
PROGRESS["output"].stem + "-failed.json"
|
||||
)
|
||||
if target.exists():
|
||||
target = target.with_name(
|
||||
target.stem + "-" + uuid4().hex[:8] + target.suffix
|
||||
)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(
|
||||
json.dumps(
|
||||
{**receipt, "result": "INCOMPLETE"},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
atexit.register(save_incomplete)
|
||||
|
||||
|
||||
def redact(text, cwd):
|
||||
directory = cwd / "secrets"
|
||||
if directory.exists():
|
||||
for path in directory.iterdir():
|
||||
if path.is_file():
|
||||
value = path.read_text().strip()
|
||||
if len(value) >= 12:
|
||||
text = text.replace(value, "<secret>")
|
||||
return (
|
||||
text.replace(str(cwd), "<clean-clone>")
|
||||
.replace(str(ROOT), "<repository>")
|
||||
.replace(str(Path.home()), "<home>")
|
||||
)
|
||||
|
||||
|
||||
def run(args, *, cwd, env, expected=0):
|
||||
print("CHECK: " + redact(" ".join(args[:3]), cwd), flush=True)
|
||||
result = subprocess.run(
|
||||
args, cwd=cwd, env=env, capture_output=True, text=True
|
||||
)
|
||||
|
||||
if result.returncode != expected:
|
||||
if "receipt" in PROGRESS:
|
||||
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args), cwd)
|
||||
raise RuntimeError(redact(result.stdout + result.stderr, cwd))
|
||||
return redact(result.stdout.strip(), cwd)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/clean-clone.json")
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
if output.exists():
|
||||
raise SystemExit("Choose new output path")
|
||||
assert not subprocess.check_output(
|
||||
["git", "status", "--porcelain"], cwd=ROOT
|
||||
)
|
||||
sha = subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
|
||||
).strip()
|
||||
identity = uuid4().hex[:10]
|
||||
project = "mtusi-lab6-clean-" + identity
|
||||
with socket.socket() as reservation:
|
||||
reservation.bind(("127.0.0.1", 0))
|
||||
public_port = str(reservation.getsockname()[1])
|
||||
with socket.socket() as grafana_reservation:
|
||||
grafana_reservation.bind(("127.0.0.1", 0))
|
||||
grafana_port = str(grafana_reservation.getsockname()[1])
|
||||
env = {
|
||||
**os.environ,
|
||||
"COMPOSE_PROJECT_NAME": project,
|
||||
"API_IMAGE": "mtusi-messenger:lab6-clean-" + identity,
|
||||
"PUBLIC_PORT": public_port,
|
||||
"GRAFANA_PORT": grafana_port,
|
||||
"VCS_REF": sha,
|
||||
"COMPOSE_FILE": "compose.yaml:compose.test.yaml",
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
receipt = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": sha,
|
||||
"fresh_secrets_environment_and_volumes": True,
|
||||
"command": "python scripts/verify_clean.py",
|
||||
"fixed_host_port_for_restart": int(public_port),
|
||||
}
|
||||
PROGRESS.update(receipt=receipt, output=output)
|
||||
with tempfile.TemporaryDirectory(prefix="mtusi-lab6-clean-") as temporary:
|
||||
clone = Path(temporary) / "lab6"
|
||||
run(
|
||||
[
|
||||
"git",
|
||||
"clone",
|
||||
"--no-hardlinks",
|
||||
"--branch",
|
||||
"solution/bvt2301-maslov",
|
||||
str(ROOT),
|
||||
str(clone),
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
)
|
||||
for name in [".env", ".venv", "secrets"]:
|
||||
assert not (clone / name).exists()
|
||||
run(["uv", "sync", "--locked"], cwd=clone, env=env)
|
||||
receipt["static_checks"] = run(
|
||||
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env)
|
||||
run(["make", "bootstrap"], cwd=clone, env=env)
|
||||
try:
|
||||
run(
|
||||
["docker", "compose", "up", "-d", "--build", "--wait"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
env["BASE_URL"] = "http://127.0.0.1:" + public_port
|
||||
receipt["smoke"] = run(
|
||||
["make", "test", "BASE_URL=" + env["BASE_URL"]],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["http_tests"] = run(
|
||||
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
|
||||
)
|
||||
for module, extra in [
|
||||
("verify_telemetry", []),
|
||||
(
|
||||
"verify_observability_persistence",
|
||||
["--causal-evidence", "evidence/clean-telemetry.json"],
|
||||
),
|
||||
]:
|
||||
name = (
|
||||
"telemetry"
|
||||
if module == "verify_telemetry"
|
||||
else "observability-persistence"
|
||||
)
|
||||
run(
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts." + module,
|
||||
"--output",
|
||||
"evidence/clean-" + name + ".json",
|
||||
*extra,
|
||||
],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt[name] = json.loads(
|
||||
(clone / ("evidence/clean-" + name + ".json")).read_text()
|
||||
)
|
||||
run(
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cluster",
|
||||
"--output",
|
||||
"evidence/clean-cluster.json",
|
||||
],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
for command_args in [
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cleanup",
|
||||
"seed",
|
||||
"--seed",
|
||||
"evidence/clean-gc-seed.json",
|
||||
],
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_pipeline",
|
||||
"--output",
|
||||
"evidence/clean-pipeline.json",
|
||||
],
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cleanup",
|
||||
"verify",
|
||||
"--seed",
|
||||
"evidence/clean-gc-seed.json",
|
||||
"--output",
|
||||
"evidence/clean-cleanup.json",
|
||||
],
|
||||
]:
|
||||
run(command_args, cwd=clone, env=env)
|
||||
receipt["pipeline"] = json.loads(
|
||||
(clone / "evidence/clean-pipeline.json").read_text()
|
||||
)
|
||||
receipt["cleanup"] = json.loads(
|
||||
(clone / "evidence/clean-cleanup.json").read_text()
|
||||
)
|
||||
receipt["cluster"] = json.loads(
|
||||
(clone / "evidence/clean-cluster.json").read_text()
|
||||
)
|
||||
import hashlib
|
||||
|
||||
expected = {
|
||||
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in (clone / "src").glob("*.py")
|
||||
}
|
||||
ids = run(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
|
||||
).split()
|
||||
assert len(ids) == 2
|
||||
code = (
|
||||
"import hashlib,json,pathlib; print(json.dumps({p.name:"
|
||||
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
|
||||
)
|
||||
ids += run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"ps",
|
||||
"-q",
|
||||
"worker",
|
||||
"relay",
|
||||
"reconciler",
|
||||
"controller",
|
||||
"observer",
|
||||
],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
).split()
|
||||
assert len(ids) == 7
|
||||
for cid in ids:
|
||||
actual = json.loads(
|
||||
run(
|
||||
["docker", "exec", cid, "python", "-c", code],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
)
|
||||
assert actual == expected
|
||||
info = json.loads(
|
||||
run(["docker", "inspect", cid], cwd=clone, env=env)
|
||||
)[0]
|
||||
assert (
|
||||
info["Config"]["Labels"][
|
||||
"org.opencontainers.image.revision"
|
||||
]
|
||||
== sha
|
||||
)
|
||||
assert not info["HostConfig"]["PortBindings"]
|
||||
receipt[
|
||||
"all_seven_application_roles_runtime_hashes_and_revision_match"
|
||||
] = True
|
||||
receipt["fsck"] = run(
|
||||
["git", "fsck", "--full"], cwd=clone, env=env
|
||||
)
|
||||
receipt["checks_completed"] = True
|
||||
except BaseException:
|
||||
receipt["partial_evidence"] = {}
|
||||
for name in [
|
||||
"telemetry",
|
||||
"observability-persistence",
|
||||
"cluster",
|
||||
"pipeline",
|
||||
"cleanup",
|
||||
"gc-seed",
|
||||
]:
|
||||
partial = clone / "evidence" / ("clean-" + name + ".json")
|
||||
if partial.exists():
|
||||
receipt["partial_evidence"][name] = json.loads(
|
||||
redact(partial.read_text(), clone)
|
||||
)
|
||||
receipt["failure_diagnostics"] = {}
|
||||
for label, arguments in [
|
||||
("services", ["ps", "-a", "--format", "json"]),
|
||||
(
|
||||
"logs",
|
||||
[
|
||||
"logs",
|
||||
"--no-color",
|
||||
"--tail",
|
||||
"60",
|
||||
"api",
|
||||
"s3",
|
||||
"migrate",
|
||||
"cache",
|
||||
],
|
||||
),
|
||||
]:
|
||||
diagnostic = subprocess.run(
|
||||
["docker", "compose", *arguments],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
receipt["failure_diagnostics"][label] = redact(
|
||||
diagnostic.stdout + diagnostic.stderr, clone
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
assert project.startswith("mtusi-lab6-clean-")
|
||||
run(
|
||||
["docker", "compose", "down", "--volumes", "--remove-orphans"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["result"] = "PASS"
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Real S3 retention test: seed is separate from verification output."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
||||
from scripts.verify_pipeline import inside, operator, wait
|
||||
|
||||
|
||||
def seed(path):
|
||||
if path.exists():
|
||||
raise ValueError("Choose a new seed file")
|
||||
ready()
|
||||
compose("stop", "--timeout", "1", "reconciler")
|
||||
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c:
|
||||
user, _, session = actor(c)
|
||||
headers = auth(session)
|
||||
result = c.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers,
|
||||
json={
|
||||
"title": "Cleanup synthetic seed",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert result.status_code == 201
|
||||
chat = result.json()["id"]
|
||||
r = c.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
headers={**headers, "Content-Type": "image/png"},
|
||||
params={"filename": "retained.png"},
|
||||
content=raw,
|
||||
)
|
||||
assert r.status_code == 202
|
||||
attachment = r.json()["id"]
|
||||
wait(
|
||||
lambda: (
|
||||
c.get(
|
||||
"/api/v1/attachments/" + attachment, headers=headers
|
||||
).json()["status"]
|
||||
== "ready"
|
||||
)
|
||||
)
|
||||
job = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
|
||||
)
|
||||
data = {
|
||||
"seed_id": uuid4().hex,
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"ready_attachment": attachment,
|
||||
"ready_job": job,
|
||||
"chat": chat,
|
||||
"owner": user["id"],
|
||||
}
|
||||
# Real old v1 objects will become obsolete after v2 is ready.
|
||||
second = operator(job, version=2)
|
||||
wait(
|
||||
lambda: inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))"
|
||||
)
|
||||
)
|
||||
data["version2_job"] = second
|
||||
data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan"
|
||||
data["failed_id"] = str(uuid4())
|
||||
data["active_id"] = str(uuid4())
|
||||
data["reservation_id"] = str(uuid4())
|
||||
data["expired_reservation_id"] = str(uuid4())
|
||||
data["failed_source"] = f"objects/staging/{data['failed_id']}/original"
|
||||
data["active_source"] = f"objects/staging/{data['active_id']}/original"
|
||||
data["reservation_key"] = (
|
||||
f"objects/staging/{data['reservation_id']}/original"
|
||||
)
|
||||
data["expired_reservation_key"] = (
|
||||
f"objects/staging/{data['expired_reservation_id']}/original"
|
||||
)
|
||||
inside(f"""from pathlib import Path
|
||||
raw=Path('/app/src/__init__.py').read_bytes() or b'fixture'
|
||||
keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])}
|
||||
for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest())
|
||||
with db.connection() as c, c.transaction():
|
||||
for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]:
|
||||
key='objects/staging/'+identity+'/original'
|
||||
c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None))
|
||||
c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None))
|
||||
c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r}))
|
||||
print('true')""")
|
||||
data["seeded_unix"] = time.time()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: retention seed created; real S3 age must reach300s", flush=True
|
||||
)
|
||||
|
||||
|
||||
def verify(seed_path, output):
|
||||
if output.exists():
|
||||
raise ValueError("Choose a new output path")
|
||||
data = json.loads(seed_path.read_text())
|
||||
report = {
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"seed": seed_path.name,
|
||||
"complete": False,
|
||||
}
|
||||
try:
|
||||
compose("stop", "--timeout", "1", "reconciler")
|
||||
|
||||
def age():
|
||||
return inside(
|
||||
f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))"
|
||||
)
|
||||
|
||||
while age() < 302:
|
||||
time.sleep(min(5, 303 - age()))
|
||||
report["real_s3_age_seconds"] = age()
|
||||
fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan"
|
||||
inside(
|
||||
f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')"
|
||||
)
|
||||
oldkeys = [
|
||||
data["orphan"],
|
||||
data["failed_source"],
|
||||
data["expired_reservation_key"],
|
||||
] + [
|
||||
f"objects/{data['ready_attachment']}/v1/{v}"
|
||||
for v in ["original", "thumbnail", "crop", "optimized"]
|
||||
]
|
||||
protected = [data["active_source"], data["reservation_key"], fresh] + [
|
||||
f"objects/{data['ready_attachment']}/v2/{v}"
|
||||
for v in ["original", "thumbnail", "crop", "optimized"]
|
||||
]
|
||||
before = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert all(before.values())
|
||||
dry = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=False,grace=300)))"
|
||||
)
|
||||
candidates = {a["key"] for a in dry["actions"]}
|
||||
assert set(oldkeys) <= candidates and not set(protected) & candidates
|
||||
afterdry = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert afterdry == before
|
||||
applied = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
|
||||
)
|
||||
repeat = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
|
||||
)
|
||||
assert not repeat["actions"]
|
||||
after = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert all(not after[k] for k in oldkeys) and all(
|
||||
after[k] for k in protected
|
||||
)
|
||||
failed = inside(
|
||||
f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))"
|
||||
)
|
||||
assert failed["error_code"] == "source_expired"
|
||||
try:
|
||||
operator(failed["id"])
|
||||
except RuntimeError:
|
||||
replay_refused = True
|
||||
else:
|
||||
raise AssertionError("Expired source replay must fail")
|
||||
# Expired reservation was removed, so a delayed API cannot commit its handoff.
|
||||
fenced = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))"
|
||||
)
|
||||
assert fenced
|
||||
report.update(
|
||||
dry_run=dry,
|
||||
apply=applied,
|
||||
repeat=repeat,
|
||||
before=before,
|
||||
after=after,
|
||||
dry_run_unchanged=True,
|
||||
active_current_and_fresh_protected=True,
|
||||
expired_source_replay_refused=replay_refused,
|
||||
expired_writer_fenced=fenced,
|
||||
complete=True,
|
||||
)
|
||||
finally:
|
||||
# Remove only this fixture's artificial active reservation/job; never real jobs.
|
||||
inside(
|
||||
f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')"
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "reconciler")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(
|
||||
"PASS: real retention, dry-run/apply/repeat, current/active/fresh protection",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("action", choices=["seed", "verify"])
|
||||
p.add_argument("--seed", required=True, type=Path)
|
||||
p.add_argument("--output", type=Path)
|
||||
a = p.parse_args()
|
||||
if a.action == "seed":
|
||||
seed(a.seed)
|
||||
else:
|
||||
if a.output is None:
|
||||
p.error("output required")
|
||||
verify(a.seed, a.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,347 @@
|
||||
"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts import cluster
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = compose("ps", "-q").split()
|
||||
result = []
|
||||
for c in json.loads(command(["docker", "inspect", *ids])):
|
||||
role = c["Config"]["Labels"]["com.docker.compose.service"]
|
||||
entry = {
|
||||
"role": role,
|
||||
"container": c["Id"][:12],
|
||||
"pid": c["State"]["Pid"],
|
||||
"started_at": c["State"]["StartedAt"],
|
||||
"ports": c["HostConfig"]["PortBindings"],
|
||||
"cpu_nano": c["HostConfig"]["NanoCpus"],
|
||||
"memory_limit": c["HostConfig"]["Memory"],
|
||||
}
|
||||
if role == "api":
|
||||
entry["instance"] = cluster.alias(c["Id"])
|
||||
assert not entry["ports"]
|
||||
result.append(entry)
|
||||
return result
|
||||
|
||||
|
||||
def exercise(client, session, chat, action, *, seconds=15):
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
observations, confirmed, errors, reads = [], {}, 0, 0
|
||||
started = time.monotonic()
|
||||
finished = threading.Event()
|
||||
action_result = {}
|
||||
|
||||
def operate():
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
action()
|
||||
except BaseException as exc:
|
||||
action_result["error"] = type(exc).__name__
|
||||
finally:
|
||||
action_result["finished"] = round(time.monotonic() - started, 3)
|
||||
finished.set()
|
||||
|
||||
thread = threading.Thread(target=operate)
|
||||
thread.start()
|
||||
key = uuid4().hex
|
||||
try:
|
||||
while (
|
||||
time.monotonic() - started < seconds
|
||||
or not finished.is_set()
|
||||
or time.monotonic() - started
|
||||
< action_result.get("finished", 0) + 2
|
||||
):
|
||||
h = {**auth(session), "Idempotency-Key": key}
|
||||
try:
|
||||
r = client.post(path, headers=h, json={"text": key})
|
||||
if r.status_code != 201:
|
||||
errors += 1
|
||||
continue
|
||||
m = r.json()
|
||||
confirmed[m["id"]] = m
|
||||
observations.append(
|
||||
{
|
||||
"at": round(time.monotonic() - started, 3),
|
||||
"instance": r.headers["x-instance-id"],
|
||||
}
|
||||
)
|
||||
key = uuid4().hex
|
||||
read = client.get(
|
||||
path, headers=auth(session), params={"limit": 100}
|
||||
)
|
||||
reads += read.status_code == 200
|
||||
errors += read.status_code != 200
|
||||
except httpx.HTTPError:
|
||||
errors += 1
|
||||
time.sleep(0.04)
|
||||
finally:
|
||||
thread.join(timeout=90)
|
||||
assert not action_result.get("error"), action_result
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
r = client.get(
|
||||
path,
|
||||
headers=auth(session),
|
||||
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
page = r.json()
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
by_id = {m["id"]: m for m in found}
|
||||
assert all(by_id.get(k) == v for k, v in confirmed.items())
|
||||
texts = [m["text"] for m in found]
|
||||
assert len(texts) == len(set(texts))
|
||||
recovery = next(
|
||||
(
|
||||
x["at"]
|
||||
for x in observations
|
||||
if x["at"] >= action_result["finished"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
assert recovery is not None and (seconds > 20 or recovery <= 10)
|
||||
elapsed = time.monotonic() - started
|
||||
return {
|
||||
"confirmed_writes": len(confirmed),
|
||||
"successful_reads": reads,
|
||||
"errors": errors,
|
||||
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
|
||||
"first_success_after_action_seconds": recovery,
|
||||
"action": action_result,
|
||||
"observations": observations,
|
||||
"confirmed_preserved": True,
|
||||
"duplicate_texts": False,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/cluster-repeat.json")
|
||||
args = parser.parse_args()
|
||||
report = {
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"complete": False,
|
||||
}
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
ready()
|
||||
initial = snapshot()
|
||||
apis = [c for c in initial if c["role"] == "api"]
|
||||
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
|
||||
report["before"] = initial
|
||||
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
|
||||
user, _, session = actor(client)
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={
|
||||
"title": "Cluster verification",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
chat = r.json()
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = uuid4().hex
|
||||
|
||||
def repeat(_):
|
||||
with httpx.Client(
|
||||
base_url=BASE, trust_env=False, timeout=8
|
||||
) as c:
|
||||
r = c.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": key},
|
||||
json={"text": "Same concurrent request"},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
return r.json(), r.headers["x-instance-id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
repeated = list(pool.map(repeat, range(30)))
|
||||
assert len({m[0]["id"] for m in repeated}) == 1
|
||||
assert len({m[1] for m in repeated}) == 2
|
||||
report["parallel_replay"] = {
|
||||
"requests": 30,
|
||||
"messages": 1,
|
||||
"instances": sorted({m[1] for m in repeated}),
|
||||
}
|
||||
# Сбой второй записи транзакции не должен оставить сообщение.
|
||||
fault_key = uuid4().hex
|
||||
fault_text = "Atomic rollback " + fault_key
|
||||
sql(
|
||||
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
|
||||
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
|
||||
+ fault_key
|
||||
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
|
||||
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
|
||||
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
|
||||
)
|
||||
try:
|
||||
failed = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": fault_text},
|
||||
)
|
||||
assert failed.status_code == 500
|
||||
assert (
|
||||
sql(
|
||||
"SELECT count(*) FROM messages WHERE text='"
|
||||
+ fault_text
|
||||
+ "'"
|
||||
)
|
||||
== "0"
|
||||
)
|
||||
finally:
|
||||
sql(
|
||||
"DROP TRIGGER student_idempo_fault ON idempotency; "
|
||||
"DROP FUNCTION student_idempo_fault()"
|
||||
)
|
||||
retried = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": fault_text},
|
||||
)
|
||||
assert retried.status_code == 201
|
||||
report["atomic_rollback_and_5xx_retry"] = True
|
||||
sql(
|
||||
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
|
||||
"WHERE key='" + fault_key + "'"
|
||||
)
|
||||
expired = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": "New retention window"},
|
||||
)
|
||||
assert expired.status_code == 201
|
||||
assert expired.json()["id"] != retried.json()["id"]
|
||||
report["expired_key_can_be_reused"] = True
|
||||
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
|
||||
report["failovers"] = []
|
||||
for api in apis:
|
||||
|
||||
def stop(target=api["instance"]):
|
||||
env = {**os.environ, "TARGET_INSTANCE_ID": target}
|
||||
subprocess.run(
|
||||
["scripts/stop-one.sh"],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={"title": "Failover", "member_ids": [user["id"]]},
|
||||
)
|
||||
result = exercise(client, session, r.json(), stop)
|
||||
assert api["instance"] not in [
|
||||
o["instance"] for o in result["observations"][-5:]
|
||||
]
|
||||
assert (
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"-f",
|
||||
"{{.State.Running}}",
|
||||
api["container"],
|
||||
]
|
||||
)
|
||||
== "false"
|
||||
)
|
||||
result["stopped"] = api
|
||||
report["failovers"].append(result)
|
||||
compose("up", "-d", "--no-build", "--wait")
|
||||
ready()
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={"title": "Rolling", "member_ids": [user["id"]]},
|
||||
)
|
||||
before_roll = snapshot()
|
||||
report["rolling"] = exercise(
|
||||
client,
|
||||
session,
|
||||
r.json(),
|
||||
lambda: command(["scripts/rolling-update.sh"]),
|
||||
seconds=45,
|
||||
)
|
||||
after_roll = snapshot()
|
||||
assert not (
|
||||
{c["container"] for c in before_roll if c["role"] == "api"}
|
||||
& {c["container"] for c in after_roll if c["role"] == "api"}
|
||||
)
|
||||
report["rolling"]["before"] = before_roll
|
||||
report["rolling"]["after"] = after_roll
|
||||
replay = repeat(0)[0]
|
||||
assert replay == repeated[0][0]
|
||||
report["replay_after_replacement"] = True
|
||||
before = snapshot()
|
||||
env = {
|
||||
**os.environ,
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
for mode, hook in [
|
||||
("failover", "scripts/stop-one.sh"),
|
||||
("persistence", "scripts/restart.sh"),
|
||||
]:
|
||||
if mode == "persistence":
|
||||
compose("up", "-d", "--no-build", "--wait")
|
||||
ready()
|
||||
before = snapshot()
|
||||
run = subprocess.run(
|
||||
[
|
||||
".venv/bin/python",
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
mode,
|
||||
"--action-script",
|
||||
hook,
|
||||
"--base-url",
|
||||
BASE,
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert run.returncode == 0, run.stdout + run.stderr
|
||||
report["public_" + mode] = run.stdout.strip()
|
||||
after = snapshot()
|
||||
for role in ["postgres", "cache", "s3", "broker", "lb"]:
|
||||
old = next(c for c in before if c["role"] == role)
|
||||
new = next(c for c in after if c["role"] == role)
|
||||
assert old["started_at"] != new["started_at"]
|
||||
report["after_persistence"] = after
|
||||
report["complete"] = True
|
||||
finally:
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS: process failover, rolling, replay and full persistence")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Restart actual stores and compare data accepted before the restart."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import httpx
|
||||
|
||||
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
||||
from scripts.observe import fetch, logs, query, spans, wait
|
||||
from scripts.stand import ROOT, command, compose
|
||||
|
||||
|
||||
def instances():
|
||||
result = {}
|
||||
for role in ["prometheus", "loki", "tempo", "grafana"]:
|
||||
cid = compose("ps", "-q", role)
|
||||
data = json.loads(command(["docker", "inspect", cid]))[0]
|
||||
result[role] = {
|
||||
"started_at": data["State"]["StartedAt"],
|
||||
"volumes": sorted(
|
||||
m["Name"] for m in data["Mounts"] if m["Type"] == "volume"
|
||||
),
|
||||
"read_only": data["HostConfig"]["ReadonlyRootfs"],
|
||||
"cap_drop": data["HostConfig"]["CapDrop"],
|
||||
"user": data["Config"]["User"],
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--causal-evidence", default="evidence/telemetry-first.json"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", default="evidence/observability-persistence-first.json"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
assert not output.exists()
|
||||
causal = json.loads((ROOT / args.causal_evidence).read_text())
|
||||
assert causal["complete"]
|
||||
identity = causal["causal_uploads"][0]["trace_id"]
|
||||
timestamp = time.time() - 20
|
||||
historical = query("sum(messenger_http_requests_total)", at=timestamp)
|
||||
assert historical and spans(identity) and logs(identity)["data"]["result"]
|
||||
report = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"trace_id": identity,
|
||||
"historical_query_at": timestamp,
|
||||
"historical_before": historical,
|
||||
"before": instances(),
|
||||
"complete": False,
|
||||
}
|
||||
password = (ROOT / "secrets/grafana_password").read_text().strip()
|
||||
base = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006")
|
||||
with httpx.Client(
|
||||
base_url=base, auth=("admin", password), timeout=10, trust_env=False
|
||||
) as grafana:
|
||||
preferences = grafana.get("/api/user/preferences")
|
||||
preferences.raise_for_status()
|
||||
previous = preferences.json()
|
||||
temporary = {
|
||||
**previous,
|
||||
"theme": "light" if previous.get("theme") != "light" else "dark",
|
||||
}
|
||||
try:
|
||||
changed = grafana.put("/api/user/preferences", json=temporary)
|
||||
changed.raise_for_status()
|
||||
compose("restart", "prometheus", "loki", "tempo", "grafana")
|
||||
for role, path in [
|
||||
("prometheus", "/-/ready"),
|
||||
("loki", "/ready"),
|
||||
("tempo", "/ready"),
|
||||
]:
|
||||
wait(
|
||||
lambda role=role, path=path: fetch(
|
||||
role, path, json_result=False
|
||||
),
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
def grafana_ready():
|
||||
try:
|
||||
response = grafana.get("/api/health")
|
||||
return (
|
||||
response.status_code == 200
|
||||
and response.json()["database"] == "ok"
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
return False
|
||||
|
||||
wait(grafana_ready, timeout=120)
|
||||
wait(lambda: spans(identity), timeout=120)
|
||||
wait(lambda: logs(identity)["data"]["result"], timeout=120)
|
||||
report["historical_after"] = query(
|
||||
"sum(messenger_http_requests_total)", at=timestamp
|
||||
)
|
||||
assert report["historical_after"] == historical
|
||||
assert (
|
||||
grafana.get("/api/user/preferences").json()["theme"]
|
||||
== temporary["theme"]
|
||||
)
|
||||
report["after"] = instances()
|
||||
for role, before in report["before"].items():
|
||||
after = report["after"][role]
|
||||
assert before["started_at"] != after["started_at"]
|
||||
assert (
|
||||
before["volumes"] == after["volumes"] and before["volumes"]
|
||||
)
|
||||
assert after["read_only"] and after["user"] not in {
|
||||
"",
|
||||
"0",
|
||||
"0:0",
|
||||
"root",
|
||||
}
|
||||
assert "ALL" in after["cap_drop"]
|
||||
report.update(
|
||||
stored_trace_survives=True,
|
||||
stored_logs_survive=True,
|
||||
historical_metric_survives=True,
|
||||
grafana_sqlite_preference_survives=True,
|
||||
complete=True,
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
grafana.put(
|
||||
"/api/user/preferences", json=previous
|
||||
).raise_for_status()
|
||||
finally:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(
|
||||
"PASS: metrics, traces, logs and Grafana state survive real store restarts"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,687 @@
|
||||
"""Repeatable fault injection for grade 5, track 2; synthetic data only."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
||||
os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
|
||||
os.environ.setdefault("no_proxy", "localhost,127.0.0.1")
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
||||
|
||||
PREFIX = """import io,json,hashlib,time,urllib.request
|
||||
from uuid import UUID,uuid4
|
||||
from psycopg.types.json import Jsonb
|
||||
from src.pipeline import database,broker,envelope,STREAM,GROUP
|
||||
from src.object_store import ObjectStore,BUCKET
|
||||
from src.operator import change
|
||||
from src.reconcile import reconcile
|
||||
db,store,queue=database(),ObjectStore(),broker()
|
||||
"""
|
||||
|
||||
|
||||
def inside(code):
|
||||
cid = compose("ps", "-q", "controller").split()[0]
|
||||
return json.loads(
|
||||
command(
|
||||
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def wait(fn, timeout=90):
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
value = fn()
|
||||
if value:
|
||||
return value
|
||||
time.sleep(0.15)
|
||||
raise TimeoutError("Fault scenario condition not reached")
|
||||
|
||||
|
||||
def stop(role):
|
||||
compose("stop", "--timeout", "1", role)
|
||||
|
||||
|
||||
def start(role):
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", role)
|
||||
|
||||
|
||||
def row(job):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))"
|
||||
)
|
||||
|
||||
|
||||
def inject(job, phase, action, remaining=1):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))"
|
||||
)
|
||||
|
||||
|
||||
def hit(identity):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))"
|
||||
)
|
||||
|
||||
|
||||
def operator(job, version=None):
|
||||
return inside(
|
||||
f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))"
|
||||
)
|
||||
|
||||
|
||||
def duplicate(job, count=5):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))"
|
||||
)
|
||||
|
||||
|
||||
def inspect_effect(job, *, pending=False):
|
||||
return inside(f"""with db.connection() as c:
|
||||
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
|
||||
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
|
||||
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
|
||||
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
|
||||
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
|
||||
pending_entries=[]
|
||||
if {pending!r}:
|
||||
for item in queue.xpending_range(STREAM,GROUP,'-','+',1000):
|
||||
entry=item['message_id']
|
||||
messages=queue.xrange(STREAM,min=entry,max=entry)
|
||||
if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry)
|
||||
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/pipeline-repeat.json")
|
||||
parser.add_argument("--repeats", type=int, default=2)
|
||||
parser.add_argument(
|
||||
"--start-at", choices=["all", "versions"], default="all"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
if output.exists():
|
||||
raise ValueError("Choose a new output path")
|
||||
report = {
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"track": 2,
|
||||
"scope": args.start_at,
|
||||
"complete": False,
|
||||
"cases": [],
|
||||
}
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def save(name, details):
|
||||
report["cases"].append({"case": name, **details})
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS:", name, flush=True)
|
||||
|
||||
try:
|
||||
ready()
|
||||
# Check instrumentation is really enabled and production default is off.
|
||||
cid = compose("ps", "-q", "api").split()[0]
|
||||
assert (
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"python",
|
||||
"-c",
|
||||
"import os;print(os.getenv('PIPELINE_FAULTS'))",
|
||||
]
|
||||
)
|
||||
== "1"
|
||||
)
|
||||
stop("reconciler")
|
||||
with httpx.Client(
|
||||
base_url=BASE, trust_env=False, timeout=25
|
||||
) as client:
|
||||
user, _, session = actor(client)
|
||||
h = auth(session)
|
||||
c = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=h,
|
||||
json={
|
||||
"title": "Pipeline synthetic faults",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert c.status_code == 201
|
||||
chat = c.json()["id"]
|
||||
raw_io = io.BytesIO()
|
||||
Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save(
|
||||
raw_io, format="PNG"
|
||||
)
|
||||
raw = raw_io.getvalue()
|
||||
|
||||
def upload():
|
||||
r = client.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
headers={**h, "Content-Type": "image/png"},
|
||||
params={"filename": "synthetic.png"},
|
||||
content=raw,
|
||||
)
|
||||
assert r.status_code == 202, r.text
|
||||
a = r.json()
|
||||
assert a["status"] == "queued" and not a["variants"]
|
||||
j = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
|
||||
)
|
||||
return a, j
|
||||
|
||||
def metadata(a):
|
||||
r = client.get("/api/v1/attachments/" + a["id"], headers=h)
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()
|
||||
|
||||
def until(a, state="ready", timeout=90):
|
||||
return wait(
|
||||
lambda: (
|
||||
r if (r := metadata(a))["status"] == state else None
|
||||
),
|
||||
timeout,
|
||||
)
|
||||
|
||||
def confirm(a, j):
|
||||
until(a)
|
||||
downloaded = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
)
|
||||
assert (
|
||||
downloaded.status_code == 200 and downloaded.content == raw
|
||||
)
|
||||
m = metadata(a)
|
||||
assert (
|
||||
m["sha256"] == hashlib.sha256(raw).hexdigest()
|
||||
and m["error_code"] is None
|
||||
)
|
||||
assert {v["name"] for v in m["variants"]} == {
|
||||
"thumbnail",
|
||||
"crop",
|
||||
"optimized",
|
||||
}
|
||||
for v in m["variants"]:
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content",
|
||||
headers=h,
|
||||
params={"variant": v["name"]},
|
||||
)
|
||||
assert (
|
||||
r.status_code == 200
|
||||
and hashlib.sha256(r.content).hexdigest()
|
||||
== v["sha256"]
|
||||
)
|
||||
with Image.open(io.BytesIO(r.content)) as image:
|
||||
assert (
|
||||
image.size
|
||||
== {
|
||||
"thumbnail": (256, 160),
|
||||
"crop": (128, 128),
|
||||
"optimized": (320, 200),
|
||||
}[v["name"]]
|
||||
)
|
||||
assert not image.getexif()
|
||||
effect = inspect_effect(j)
|
||||
assert (
|
||||
effect["publications"] == 1
|
||||
and len(effect["variants"]) == 4
|
||||
and len(effect["physical_keys"]) == 4
|
||||
)
|
||||
return effect
|
||||
|
||||
if args.start_at == "all":
|
||||
# Source-only crash before the DB handoff must never be reported as accepted.
|
||||
stop("worker")
|
||||
stop("relay")
|
||||
marker = inject(chat, "api_after_staging", "wait")
|
||||
before = inside(
|
||||
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
|
||||
)
|
||||
pool = ThreadPoolExecutor(max_workers=1)
|
||||
future = pool.submit(
|
||||
lambda: client.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
headers={**h, "Content-Type": "image/png"},
|
||||
params={"filename": "unconfirmed.png"},
|
||||
content=raw,
|
||||
)
|
||||
)
|
||||
wait(lambda marker=marker: hit(marker))
|
||||
compose("kill", "-s", "SIGKILL", "api")
|
||||
try:
|
||||
response = future.result(15)
|
||||
assert response.status_code != 202
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
pool.shutdown()
|
||||
start("api")
|
||||
ready()
|
||||
after = inside(
|
||||
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
|
||||
)
|
||||
assert before == after
|
||||
save(
|
||||
"staging_before_handoff_crash",
|
||||
{
|
||||
"no_202": True,
|
||||
"no_attachment_or_job_created": True,
|
||||
"orphan_retained_for_cleanup": True,
|
||||
},
|
||||
)
|
||||
# API durable commit/outbox survives restart without a broker or worker.
|
||||
stop("broker")
|
||||
a, j = upload()
|
||||
linked = client.post(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers={**h, "Idempotency-Key": uuid4().hex},
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
)
|
||||
assert linked.status_code == 201
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
).status_code
|
||||
== 409
|
||||
)
|
||||
compose("restart", "api")
|
||||
ready()
|
||||
start("broker")
|
||||
start("relay")
|
||||
start("worker")
|
||||
effect = confirm(a, j)
|
||||
save(
|
||||
"commit_before_publish_and_worker_stopped",
|
||||
{
|
||||
"attachment": a["id"],
|
||||
"effect": effect,
|
||||
"queued_link_allowed": True,
|
||||
"api_and_broker_restarted": True,
|
||||
"broker_stopped_during_upload": True,
|
||||
},
|
||||
)
|
||||
# Broker-volume survival with a confirmed, still-unconsumed stream entry.
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
wait(
|
||||
lambda: inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))"
|
||||
)
|
||||
)
|
||||
stop("relay")
|
||||
ids = inside("print(json.dumps(queue.xrange(STREAM)))")
|
||||
compose("kill", "-s", "SIGKILL", "broker")
|
||||
start("broker")
|
||||
wait(lambda: inside("print(json.dumps(queue.ping()))"))
|
||||
restored = inside("print(json.dumps(queue.xrange(STREAM)))")
|
||||
assert restored == ids, {"before": ids, "after": restored}
|
||||
start("relay")
|
||||
start("worker")
|
||||
save(
|
||||
"confirmed_broker_entry_survives_sigkill",
|
||||
{
|
||||
"effect": confirm(a, j),
|
||||
"same_stream_entries": True,
|
||||
"entry_count": len(ids),
|
||||
},
|
||||
)
|
||||
# Confirm succeeded, PG marker did not: relay recovery may publish a duplicate.
|
||||
stop("worker")
|
||||
stop("relay")
|
||||
a, j = upload()
|
||||
marker = inject(j, "relay_after_publish", "crash")
|
||||
start("relay")
|
||||
wait(lambda marker=marker: hit(marker))
|
||||
time.sleep(0.5)
|
||||
unpublished = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))"
|
||||
)
|
||||
assert unpublished >= 1
|
||||
start("relay")
|
||||
start("worker")
|
||||
save(
|
||||
"relay_crash_after_xadd_before_pg_marker",
|
||||
{
|
||||
"unpublished_after_crash": unpublished,
|
||||
"effect": confirm(a, j),
|
||||
},
|
||||
)
|
||||
# Repeated before/after effect and commit/ack windows.
|
||||
for repeat in range(args.repeats):
|
||||
for phase in [
|
||||
"before_effect",
|
||||
"after_effect",
|
||||
"after_commit",
|
||||
]:
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
marker = inject(j, phase, "wait")
|
||||
start("worker")
|
||||
wait(lambda marker=marker: hit(marker))
|
||||
at_crash = inspect_effect(
|
||||
j, pending=phase == "after_commit"
|
||||
)
|
||||
assert at_crash["job"]["state"] == (
|
||||
"ready"
|
||||
if phase == "after_commit"
|
||||
else "processing"
|
||||
)
|
||||
assert at_crash["publications"] == (
|
||||
1 if phase == "after_commit" else 0
|
||||
)
|
||||
assert len(at_crash["physical_keys"]) == (
|
||||
0 if phase == "before_effect" else 4
|
||||
)
|
||||
if phase == "after_commit":
|
||||
assert at_crash["pending_stream_entries"], (
|
||||
"Result must be committed while notification is still unacknowledged"
|
||||
)
|
||||
assert not any(
|
||||
e["event"] == "child_finished"
|
||||
for e in at_crash["events"]
|
||||
)
|
||||
compose("kill", "-s", "SIGKILL", "worker")
|
||||
start("worker")
|
||||
effect = confirm(a, j)
|
||||
if phase == "after_commit":
|
||||
wait(
|
||||
lambda j=j: any(
|
||||
e["event"] == "duplicate_decision"
|
||||
and e["detail"]["state"] == "ready"
|
||||
for e in inspect_effect(j)["events"]
|
||||
)
|
||||
)
|
||||
wait(
|
||||
lambda j=j: inside(
|
||||
f"print(json.dumps(not any(json.loads(fields.get('job','{{}}')).get('job_id')=={j!r} for _,fields in queue.xrange(STREAM))))"
|
||||
)
|
||||
)
|
||||
effect = inspect_effect(j)
|
||||
else:
|
||||
assert effect["job"]["attempt"] >= 2
|
||||
save(
|
||||
f"{phase}_sigkill_repeat_{repeat + 1}",
|
||||
{"at_crash": at_crash, "effect": effect},
|
||||
)
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
inject(j, "before_effect", "transient")
|
||||
start("worker")
|
||||
effect = confirm(a, j)
|
||||
assert effect["job"]["attempt"] == 2
|
||||
save("transient_retry_with_jitter", {"effect": effect})
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
marker = inject(j, "before_effect", "hang")
|
||||
start("worker")
|
||||
wait(lambda: hit(marker))
|
||||
limits = inside(
|
||||
f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))"
|
||||
)
|
||||
assert limits["address_space_bytes"] == [805306368, 805306368]
|
||||
assert limits["memory.max"] == "1610612736"
|
||||
effect = confirm(a, j)
|
||||
assert any(
|
||||
e["event"] == "retry"
|
||||
and e["detail"]["code"] == "worker_timeout"
|
||||
for e in effect["events"]
|
||||
)
|
||||
save(
|
||||
"hung_child_killed_and_retried",
|
||||
{"effect": effect, "observed_limits": limits},
|
||||
)
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
inject(j, "before_effect", "permanent")
|
||||
start("worker")
|
||||
failed = until(a, "failed")
|
||||
assert failed["error_code"] == "invalid_image"
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
).status_code
|
||||
== 409
|
||||
)
|
||||
r = client.post(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers={**h, "Idempotency-Key": uuid4().hex},
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
)
|
||||
assert r.status_code == 409
|
||||
assert operator(j) == j
|
||||
effect = confirm(a, j)
|
||||
assert effect["job"]["cycle"] == 2
|
||||
try:
|
||||
operator(j)
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("Replay of ready must be rejected")
|
||||
save(
|
||||
"permanent_failed_registry_and_audited_replay",
|
||||
{
|
||||
"failed": failed,
|
||||
"effect": effect,
|
||||
"ready_replay_rejected": True,
|
||||
},
|
||||
)
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
inject(j, "before_effect", "transient", remaining=5)
|
||||
start("worker")
|
||||
failed = until(a, "failed", timeout=150)
|
||||
assert (
|
||||
failed["error_code"] == "retry_exhausted"
|
||||
and row(j)["attempt"] == 5
|
||||
)
|
||||
operator(j)
|
||||
save(
|
||||
"finite_retry_exhaustion_then_explicit_replay",
|
||||
{"failed": failed, "effect": confirm(a, j)},
|
||||
)
|
||||
# Reverse notification order and add concurrent copies: one publication/key.
|
||||
stop("worker")
|
||||
stop("relay")
|
||||
a, j = upload()
|
||||
b, k = upload()
|
||||
duplicate(k, 8)
|
||||
duplicate(j, 8)
|
||||
start("worker")
|
||||
start("relay")
|
||||
ea = confirm(a, j)
|
||||
confirm(b, k)
|
||||
duplicate(j, 10)
|
||||
duplicate(k, 10)
|
||||
wait(
|
||||
lambda: any(
|
||||
e["event"] == "duplicate_decision"
|
||||
for e in inspect_effect(j)["events"]
|
||||
)
|
||||
)
|
||||
assert (
|
||||
inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"]
|
||||
)
|
||||
save(
|
||||
"reordered_and_duplicate_notifications",
|
||||
{
|
||||
"effects": [inspect_effect(j), inspect_effect(k)],
|
||||
"added_duplicates": 36,
|
||||
"no_extra_attempt_after_ready": True,
|
||||
},
|
||||
)
|
||||
# Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS.
|
||||
for repeat in range(args.repeats):
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
marker = inject(j, "after_effect", "wait")
|
||||
# Keep one controller process hot throughout the narrow stale-writer window.
|
||||
code = f"""from src.pipeline import event
|
||||
with db.connection() as c: event(c,UUID({j!r}),'version_controller_waiting')
|
||||
end=time.monotonic()+45
|
||||
while True:
|
||||
with db.connection() as c: hit=c.execute('SELECT hit_at FROM pipeline_faults WHERE id=%s',({marker},)).fetchone()['hit_at']
|
||||
if hit: break
|
||||
if time.monotonic()>end: raise TimeoutError('v1 hook never hit')
|
||||
time.sleep(.05)
|
||||
k=change(db,store,job_id=UUID({j!r}),operator='synthetic-verifier',reason='Held stale writer race',version=2)
|
||||
end=time.monotonic()+10
|
||||
while True:
|
||||
with db.connection() as c: ready=c.execute('SELECT state FROM jobs WHERE id=%s',(k,)).fetchone()['state']=='ready'
|
||||
if ready: break
|
||||
if time.monotonic()>end: raise TimeoutError('v2 did not finish inside live v1 budget')
|
||||
time.sleep(.05)
|
||||
with db.connection() as c:
|
||||
hashes=c.execute('SELECT name,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=2 ORDER BY name',({a["id"]!r},)).fetchall()
|
||||
c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))
|
||||
end=time.monotonic()+10
|
||||
while True:
|
||||
with db.connection() as c: stale=c.execute("SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='stale_completion_rejected'",({j!r},)).fetchone()['n']
|
||||
if stale: break
|
||||
if time.monotonic()>end: raise TimeoutError('Old completion was not rejected')
|
||||
time.sleep(.05)
|
||||
print(json.dumps({{'job_id':k,'hashes':{{r['name']:r['sha256'] for r in hashes}}}}))
|
||||
"""
|
||||
with ThreadPoolExecutor(max_workers=1) as hot:
|
||||
future = hot.submit(inside, code)
|
||||
wait(
|
||||
lambda j=j: inside(
|
||||
f"with db.connection() as c: n=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='version_controller_waiting'\",({j!r},)).fetchone()['n']\nprint(json.dumps(n>0))"
|
||||
)
|
||||
)
|
||||
start("worker")
|
||||
race = future.result(timeout=45)
|
||||
k = race["job_id"]
|
||||
hashes = race["hashes"]
|
||||
confirm(a, k)
|
||||
duplicate(j, 12)
|
||||
wait(
|
||||
lambda j=j: any(
|
||||
e["event"] == "duplicate_decision"
|
||||
for e in inspect_effect(j)["events"]
|
||||
)
|
||||
)
|
||||
assert hashes == {
|
||||
v["name"]: v["sha256"] for v in confirm(a, k)["variants"]
|
||||
}
|
||||
assert inspect_effect(j)["publications"] == 0
|
||||
save(
|
||||
f"stale_v1_after_v2_repeat_{repeat + 1}",
|
||||
{
|
||||
"v1": inspect_effect(j),
|
||||
"v2": inspect_effect(k),
|
||||
"new_hashes_unchanged": True,
|
||||
},
|
||||
)
|
||||
# No arbitrary URL/bytes fields in queue envelopes.
|
||||
invalid = inside(
|
||||
"bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')"
|
||||
)
|
||||
assert invalid
|
||||
wait(
|
||||
lambda: inside(
|
||||
"with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))"
|
||||
)
|
||||
)
|
||||
save(
|
||||
"malformed_notification_rejected", {"durable_diagnostic": True}
|
||||
)
|
||||
# Private bucket, bounded IAM, and download after uploader process dies.
|
||||
a, j = upload()
|
||||
confirm(a, j)
|
||||
key = f"objects/{a['id']}/v1/original"
|
||||
security = inside(f"""from minio.error import S3Error
|
||||
from urllib.error import HTTPError
|
||||
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
|
||||
except HTTPError as e: anonymous=e.code
|
||||
else: raise AssertionError('Anonymous access allowed')
|
||||
denials={{}}
|
||||
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
|
||||
try: fn()
|
||||
except S3Error as e: denials[name]=e.code
|
||||
else: raise AssertionError('IAM allowed forbidden action')
|
||||
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
|
||||
assert security["anonymous"] == 403 and set(
|
||||
security["iam"].values()
|
||||
) == {"AccessDenied"}
|
||||
selected = None
|
||||
for cid in compose("ps", "-q", "api").split():
|
||||
alias = command(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
|
||||
]
|
||||
)
|
||||
uploader = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
|
||||
)
|
||||
if alias == uploader:
|
||||
selected = cid
|
||||
assert selected
|
||||
command(["docker", "stop", "--time", "1", selected])
|
||||
ready()
|
||||
wait(
|
||||
lambda: (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
).status_code
|
||||
== 200
|
||||
)
|
||||
)
|
||||
downloaded = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
)
|
||||
assert (
|
||||
downloaded.content == raw
|
||||
and downloaded.headers["X-Instance-Id"] != uploader
|
||||
)
|
||||
start("api")
|
||||
ready()
|
||||
save(
|
||||
"private_iam_and_other_api_download",
|
||||
{
|
||||
"security": security,
|
||||
"attachment": a["id"],
|
||||
"uploader": uploader,
|
||||
"downloader": downloaded.headers["X-Instance-Id"],
|
||||
"sha256": hashlib.sha256(raw).hexdigest(),
|
||||
},
|
||||
)
|
||||
report["worker_metrics"] = inside(
|
||||
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
|
||||
)
|
||||
report["audit"] = inside(
|
||||
"with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))"
|
||||
)
|
||||
report["complete"] = True
|
||||
finally:
|
||||
for role in ["api", "broker", "relay", "worker", "reconciler"]:
|
||||
try:
|
||||
start(role)
|
||||
except Exception:
|
||||
pass
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS: versioned pipeline scope=" + args.start_at, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Actual API-to-worker causality, log correlation, provisioning and privacy."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
||||
from scripts.observe import fetch, logs, query, spans, wait
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
||||
|
||||
|
||||
def verify_chain(items, trace_id, *, retry):
|
||||
assert items and {s["trace_id"] for s in items} == {trace_id}
|
||||
by_id = {s["span_id"]: s for s in items}
|
||||
producers = [s for s in items if s["kind"] == "SPAN_KIND_PRODUCER"]
|
||||
consumers = [s for s in items if s["kind"] == "SPAN_KIND_CONSUMER"]
|
||||
processing = [s for s in items if s["name"] == "attachment.process"]
|
||||
assert producers and consumers and processing
|
||||
assert {s["service"] for s in items} >= {
|
||||
"messenger-api",
|
||||
"messenger-relay",
|
||||
"messenger-worker",
|
||||
}
|
||||
for consumer in consumers:
|
||||
assert (
|
||||
by_id[consumer["parent_span_id"]]["kind"] == "SPAN_KIND_PRODUCER"
|
||||
)
|
||||
for process in processing:
|
||||
assert by_id[process["parent_span_id"]]["kind"] == "SPAN_KIND_CONSUMER"
|
||||
assert any(
|
||||
s["name"] == "s3.put" and s["service"] == "messenger-worker"
|
||||
for s in items
|
||||
)
|
||||
assert any(
|
||||
s["name"] == "db.postgresql" and s["service"] == "messenger-worker"
|
||||
for s in items
|
||||
)
|
||||
if retry:
|
||||
assert len(processing) >= 2
|
||||
assert any(
|
||||
s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
|
||||
for s in processing
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/telemetry-first.json")
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
assert not output.exists(), "Choose a new output path"
|
||||
report = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"complete": False,
|
||||
}
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
ready(timeout=90)
|
||||
for service, path in [
|
||||
("prometheus", "/-/ready"),
|
||||
("loki", "/ready"),
|
||||
("tempo", "/ready"),
|
||||
("otel", "/"),
|
||||
]:
|
||||
wait(lambda s=service, p=path: fetch(s, p, json_result=False))
|
||||
report["backends_ready"] = True
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (320, 200), (24, 110, 153)).save(buffer, "PNG")
|
||||
raw = buffer.getvalue()
|
||||
sentinel = "PrivacyProbe-" + uuid4().hex
|
||||
with httpx.Client(
|
||||
base_url=BASE, timeout=15, trust_env=False
|
||||
) as client:
|
||||
user, _, session = actor(client)
|
||||
headers = auth(session)
|
||||
chat = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers,
|
||||
json={
|
||||
"title": "Synthetic telemetry",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
).json()["id"]
|
||||
message = client.post(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers={**headers, "Idempotency-Key": uuid4().hex},
|
||||
json={"text": sentinel},
|
||||
)
|
||||
assert message.status_code == 201
|
||||
cases = []
|
||||
for retry in [False, True]:
|
||||
if retry:
|
||||
compose("stop", "--timeout", "1", "worker")
|
||||
trace_id = uuid4().hex
|
||||
trace_header = (
|
||||
"00-" + trace_id + "-" + uuid4().hex[:16] + "-01"
|
||||
)
|
||||
accepted = client.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
params={"filename": sentinel + ".png"},
|
||||
headers={
|
||||
**headers,
|
||||
"Content-Type": "image/png",
|
||||
"traceparent": trace_header,
|
||||
},
|
||||
content=raw,
|
||||
)
|
||||
assert (
|
||||
accepted.status_code == 202
|
||||
and accepted.json()["status"] == "queued"
|
||||
)
|
||||
attachment = accepted.json()["id"]
|
||||
job = sql(
|
||||
f"SELECT id FROM jobs WHERE attachment_id='{attachment}'::uuid"
|
||||
)
|
||||
if retry:
|
||||
sql(
|
||||
f"INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES('{job}'::uuid,'before_effect','transient',1)"
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
|
||||
|
||||
def finished(identity=attachment):
|
||||
response = client.get(
|
||||
f"/api/v1/attachments/{identity}", headers=headers
|
||||
)
|
||||
return (
|
||||
response.status_code == 200
|
||||
and response.json()["status"] == "ready"
|
||||
)
|
||||
|
||||
wait(finished, timeout=90)
|
||||
original = client.get(
|
||||
f"/api/v1/attachments/{attachment}/content",
|
||||
headers=headers,
|
||||
)
|
||||
assert original.content == raw
|
||||
|
||||
def traced(identity=trace_id, retry_expected=retry):
|
||||
items = spans(identity)
|
||||
if len(
|
||||
[s for s in items if s["name"] == "attachment.process"]
|
||||
) < (2 if retry_expected else 1):
|
||||
return None
|
||||
if any(s["kind"] == "SPAN_KIND_CONSUMER" for s in items):
|
||||
# Independent export batches may arrive in a different
|
||||
# order. Require the whole chain before accepting it.
|
||||
try:
|
||||
return verify_chain(
|
||||
items, identity, retry=retry_expected
|
||||
)
|
||||
except (AssertionError, KeyError):
|
||||
return None
|
||||
|
||||
items = wait(traced, timeout=90)
|
||||
|
||||
def correlated(identity=trace_id):
|
||||
result = logs(identity)
|
||||
streams = result.get("data", {}).get("result", [])
|
||||
services = {
|
||||
x["stream"].get("service_name") for x in streams
|
||||
}
|
||||
return (
|
||||
result
|
||||
if {"messenger-api", "messenger-worker"} <= services
|
||||
else None
|
||||
)
|
||||
|
||||
correlated_logs = wait(correlated, timeout=90)
|
||||
span_ids = {s["span_id"] for s in items}
|
||||
streams = correlated_logs["data"]["result"]
|
||||
assert all(
|
||||
row["stream"].get("span_id") in span_ids for row in streams
|
||||
)
|
||||
assert any(
|
||||
row["stream"].get("request_id")
|
||||
== accepted.headers["X-Request-Id"]
|
||||
for row in streams
|
||||
)
|
||||
exported = json.dumps(
|
||||
{"spans": items, "logs": correlated_logs}
|
||||
)
|
||||
assert (
|
||||
sentinel not in exported
|
||||
and session["token"] not in exported
|
||||
)
|
||||
cases.append(
|
||||
{
|
||||
"retry": retry,
|
||||
"attachment_id": attachment,
|
||||
"job_id": job,
|
||||
"trace_id": trace_id,
|
||||
"request_id": accepted.headers["X-Request-Id"],
|
||||
"original_sha256": hashlib.sha256(raw).hexdigest(),
|
||||
"attempt": int(
|
||||
sql(
|
||||
f"SELECT attempt FROM jobs WHERE id='{job}'::uuid"
|
||||
)
|
||||
),
|
||||
"spans": items,
|
||||
"correlated_logs": correlated_logs,
|
||||
}
|
||||
)
|
||||
report["causal_uploads"] = cases
|
||||
assert cases[1]["attempt"] >= 2
|
||||
report["privacy_sentinels_absent_from_traces_and_logs"] = True
|
||||
names = wait(
|
||||
lambda: fetch("prometheus", "/api/v1/label/__name__/values")[
|
||||
"data"
|
||||
]
|
||||
)
|
||||
required = {
|
||||
"messenger_http_requests_total",
|
||||
"messenger_http_duration_seconds_bucket",
|
||||
"messenger_job_attempts_total",
|
||||
"messenger_job_duration_seconds_bucket",
|
||||
"messenger_time_to_ready_seconds_bucket",
|
||||
"messenger_queue_depth",
|
||||
"messenger_oldest_job_age_seconds",
|
||||
"messenger_db_pool_used",
|
||||
"messenger_db_pool_waiting",
|
||||
"messenger_worker_busy",
|
||||
"messenger_worker_capacity",
|
||||
"messenger_process_cpu_seconds_total",
|
||||
"messenger_process_memory_bytes",
|
||||
"messenger_slo_decisions_total",
|
||||
}
|
||||
wait(
|
||||
lambda: (
|
||||
required
|
||||
<= set(
|
||||
fetch("prometheus", "/api/v1/label/__name__/values")[
|
||||
"data"
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
names = fetch("prometheus", "/api/v1/label/__name__/values")["data"]
|
||||
report["metric_names"] = [
|
||||
n for n in names if n.startswith("messenger_")
|
||||
]
|
||||
series = fetch(
|
||||
"prometheus",
|
||||
"/api/v1/series",
|
||||
params={"match[]": '{__name__=~"messenger_.*"}'},
|
||||
)["data"]
|
||||
forbidden = {
|
||||
"user_id",
|
||||
"chat_id",
|
||||
"message_id",
|
||||
"attachment_id",
|
||||
"job_id",
|
||||
"request_id",
|
||||
"trace_id",
|
||||
"filename",
|
||||
}
|
||||
assert all(not forbidden.intersection(s) for s in series)
|
||||
assert all(sentinel not in json.dumps(s) for s in series)
|
||||
report["metric_series_count"] = len(series)
|
||||
report["metric_labels_low_cardinality"] = True
|
||||
url = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006")
|
||||
password = (ROOT / "secrets/grafana_password").read_text().strip()
|
||||
with httpx.Client(
|
||||
base_url=url, auth=("admin", password), trust_env=False, timeout=15
|
||||
) as grafana:
|
||||
dashboard = grafana.get("/api/dashboards/uid/messenger-lab6")
|
||||
dashboard.raise_for_status()
|
||||
imported = dashboard.json()["dashboard"]
|
||||
assert len(imported["panels"]) == 18
|
||||
sources = grafana.get("/api/datasources").json()
|
||||
assert {s["uid"] for s in sources} == {
|
||||
"prometheus",
|
||||
"loki",
|
||||
"tempo",
|
||||
}
|
||||
checks = {}
|
||||
for uid in ["prometheus", "loki", "tempo"]:
|
||||
r = grafana.get(f"/api/datasources/uid/{uid}/health")
|
||||
r.raise_for_status()
|
||||
assert r.json()["status"] == "OK", uid
|
||||
checks[uid] = r.json()["status"]
|
||||
result = grafana.post(
|
||||
"/api/ds/query",
|
||||
json={
|
||||
"queries": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"uid": "prometheus",
|
||||
"type": "prometheus",
|
||||
},
|
||||
"expr": "sum(messenger_http_requests_total)",
|
||||
"instant": True,
|
||||
"format": "table",
|
||||
"intervalMs": 5000,
|
||||
"maxDataPoints": 10,
|
||||
}
|
||||
],
|
||||
"from": "now-5m",
|
||||
"to": "now",
|
||||
},
|
||||
)
|
||||
result.raise_for_status()
|
||||
assert result.json()["results"]["A"]["frames"]
|
||||
report["grafana"] = {
|
||||
"dashboard_uid": imported["uid"],
|
||||
"panels": len(imported["panels"]),
|
||||
"datasources_health": checks,
|
||||
"actual_query_frames": len(
|
||||
result.json()["results"]["A"]["frames"]
|
||||
),
|
||||
}
|
||||
report["quantiles"] = {
|
||||
str(q): query(
|
||||
f"histogram_quantile({q},sum by(le)(rate(messenger_http_duration_seconds_bucket[2m])))"
|
||||
)
|
||||
for q in [0.5, 0.95, 0.99]
|
||||
}
|
||||
assert all(report["quantiles"].values())
|
||||
report["complete"] = True
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(
|
||||
"PASS: causal API/publish/consumer/child/DB/S3, retry, correlated logs, privacy and Grafana"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,44 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
|
||||
|
||||
def early_stop(signum, _frame):
|
||||
print(
|
||||
json.dumps({"event": "stopped_before_startup", "signal": signum}),
|
||||
flush=True,
|
||||
)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
signal.signal(signal.SIGTERM, early_stop)
|
||||
signal.signal(signal.SIGINT, early_stop)
|
||||
|
||||
import uvicorn # noqa: E402
|
||||
|
||||
from src.logging_config import JsonFormatter # noqa: E402
|
||||
|
||||
|
||||
def main() -> None:
|
||||
host = os.environ.get("APP_HOST", "127.0.0.1")
|
||||
port = int(os.environ.get("APP_PORT", "8080"))
|
||||
if not 1 <= port <= 65535:
|
||||
raise ValueError("APP_PORT must be between 1 and 65535")
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
logging.basicConfig(level=logging.INFO, handlers=[handler], force=True)
|
||||
uvicorn.run(
|
||||
"src.api:app",
|
||||
host=host,
|
||||
port=port,
|
||||
workers=1,
|
||||
access_log=False,
|
||||
timeout_graceful_shutdown=5,
|
||||
log_config=None,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+412
@@ -0,0 +1,412 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import tempfile
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from anyio import Semaphore, fail_after, to_thread
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from pydantic import ValidationError
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.requests import ClientDisconnect
|
||||
from starlette.responses import JSONResponse, Response, StreamingResponse
|
||||
|
||||
from src import images, telemetry
|
||||
from src.attachments import Attachments
|
||||
from src.cache import UserCache
|
||||
from src.cursor import parse_uuid
|
||||
from src.database import Database
|
||||
from src.domain import timestamp
|
||||
from src.errors import DomainError, invalid
|
||||
from src.object_store import ObjectStore
|
||||
from src.password import Passwords
|
||||
from src.repository import PostgresRepository
|
||||
from src.schemas import CreateChat, CreateMessage, CreateUser, Input
|
||||
from src.service import MessengerService
|
||||
from src.settings import Settings
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
INSTANCE_ID = uuid4().hex
|
||||
|
||||
|
||||
def response(data: object, *, status: int = 200) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
jsonable_encoder(
|
||||
data, custom_encoder={datetime: timestamp, UUID: str}
|
||||
),
|
||||
status_code=status,
|
||||
)
|
||||
|
||||
|
||||
async def body(request: Request, model: type[Input]) -> Input:
|
||||
try:
|
||||
data = await request.body()
|
||||
return model.model_validate(
|
||||
json.loads(
|
||||
data.decode("utf-8"),
|
||||
parse_constant=lambda _: (_ for _ in ()).throw(ValueError()),
|
||||
)
|
||||
)
|
||||
except ValueError, ValidationError, UnicodeError, RecursionError:
|
||||
raise invalid() from None
|
||||
|
||||
|
||||
def pagination(request: Request) -> tuple[int, str | None]:
|
||||
limits = request.query_params.getlist("limit")
|
||||
cursors = request.query_params.getlist("cursor")
|
||||
if len(limits) > 1 or len(cursors) > 1:
|
||||
raise invalid()
|
||||
raw = limits[0] if limits else "50"
|
||||
if not raw.isascii() or not raw.isdecimal():
|
||||
raise invalid()
|
||||
raw = raw.lstrip("0") or "0"
|
||||
if len(raw) > 3:
|
||||
raise invalid()
|
||||
limit = int(raw)
|
||||
if not 1 <= limit <= 100:
|
||||
raise invalid()
|
||||
return limit, cursors[0] if cursors else None
|
||||
|
||||
|
||||
def service(request: Request) -> MessengerService:
|
||||
return request.app.state.service
|
||||
|
||||
|
||||
async def run_io(function, **kwargs):
|
||||
return await to_thread.run_sync(partial(function, **kwargs))
|
||||
|
||||
|
||||
async def actor(request: Request):
|
||||
return await run_io(
|
||||
service(request).identify, value=request.headers.get("authorization")
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
telemetry.configure("messenger-api", instance=INSTANCE_ID)
|
||||
settings = Settings.load()
|
||||
database = Database(url=settings.database_url, size=settings.pool_size)
|
||||
cache = UserCache(url=settings.cache_url)
|
||||
store = ObjectStore()
|
||||
try:
|
||||
await run_io(database.open)
|
||||
passwords = await run_io(Passwords)
|
||||
app.state.database = database
|
||||
app.state.store = store
|
||||
app.state.upload_slots = Semaphore(2)
|
||||
app.state.attachments = Attachments(
|
||||
database=database, store=store, instance=INSTANCE_ID
|
||||
)
|
||||
app.state.service = MessengerService(
|
||||
repository=PostgresRepository(database=database, cache=cache),
|
||||
passwords=passwords,
|
||||
session_ttl=settings.session_ttl,
|
||||
)
|
||||
app.state.ready = True
|
||||
yield
|
||||
finally:
|
||||
app.state.ready = False
|
||||
await run_io(store.close)
|
||||
await run_io(cache.close)
|
||||
await run_io(database.close)
|
||||
await run_io(telemetry.stop)
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None
|
||||
)
|
||||
app.add_middleware(telemetry.TelemetryMiddleware)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_context(request: Request, call_next):
|
||||
request_id = str(uuid4())
|
||||
request.state.request_id = request_id
|
||||
try:
|
||||
result = await call_next(request)
|
||||
except Exception:
|
||||
logger.error(
|
||||
json.dumps({"event": "internal_error", "request_id": request_id})
|
||||
)
|
||||
result = response(
|
||||
{
|
||||
"error": {
|
||||
"code": "internal_error",
|
||||
"message": "Internal error",
|
||||
"request_id": request_id,
|
||||
}
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
result.headers["X-Request-Id"] = request_id
|
||||
result.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return result
|
||||
|
||||
|
||||
@app.exception_handler(DomainError)
|
||||
async def domain_error(request: Request, error: DomainError):
|
||||
result = response(
|
||||
{
|
||||
"error": {
|
||||
"code": error.code,
|
||||
"message": error.message,
|
||||
"request_id": request.state.request_id,
|
||||
}
|
||||
},
|
||||
status=error.status,
|
||||
)
|
||||
result.headers.update(error.headers)
|
||||
return result
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request: Request, error: HTTPException):
|
||||
code = "not_found" if error.status_code == 404 else "invalid_request"
|
||||
return response(
|
||||
{
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": "Resource not found"
|
||||
if error.status_code == 404
|
||||
else "Invalid request",
|
||||
"request_id": request.state.request_id,
|
||||
}
|
||||
},
|
||||
status=error.status_code,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
async def live():
|
||||
return response({"status": "ok"})
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
async def ready():
|
||||
if not getattr(app.state, "ready", False):
|
||||
raise DomainError(503, "unavailable", "Not ready")
|
||||
await run_io(app.state.database.ping)
|
||||
await run_io(app.state.store.ping)
|
||||
return response({"status": "ok"})
|
||||
|
||||
|
||||
@app.post("/api/v1/users")
|
||||
async def register(request: Request):
|
||||
data = await body(request, CreateUser)
|
||||
return response(
|
||||
await run_io(service(request).register, **data.model_dump()),
|
||||
status=201,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
async def current(request: Request):
|
||||
return response(await actor(request))
|
||||
|
||||
|
||||
@app.post("/api/v1/chats")
|
||||
async def create_chat(request: Request):
|
||||
user = await actor(request)
|
||||
data = await body(request, CreateChat)
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).create_chat, actor_id=user.id, **data.model_dump()
|
||||
),
|
||||
status=201,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
async def list_chats(request: Request):
|
||||
user = await actor(request)
|
||||
limit, cursor = pagination(request)
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).chats,
|
||||
actor_id=user.id,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
async def get_chat(request: Request, chat_id: str):
|
||||
user = await actor(request)
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).get_chat,
|
||||
actor_id=user.id,
|
||||
chat_id=parse_uuid(chat_id),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages")
|
||||
async def send(request: Request, chat_id: str):
|
||||
user = await actor(request)
|
||||
cid = parse_uuid(chat_id)
|
||||
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
|
||||
data = await body(request, CreateMessage)
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).send,
|
||||
actor_id=user.id,
|
||||
chat_id=cid,
|
||||
idempotency_key=request.headers.get("idempotency-key"),
|
||||
**data.model_dump(),
|
||||
),
|
||||
status=201,
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
async def history(request: Request, chat_id: str):
|
||||
user = await actor(request)
|
||||
cid = parse_uuid(chat_id)
|
||||
chat = await run_io(
|
||||
service(request).get_chat, actor_id=user.id, chat_id=cid
|
||||
)
|
||||
limit, cursor = pagination(request)
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).history,
|
||||
actor_id=user.id,
|
||||
chat=chat,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions")
|
||||
async def login(request: Request):
|
||||
return response(
|
||||
await run_io(
|
||||
service(request).login,
|
||||
authorization=request.headers.get("authorization"),
|
||||
),
|
||||
status=201,
|
||||
)
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current")
|
||||
async def logout(request: Request):
|
||||
await run_io(
|
||||
service(request).logout,
|
||||
authorization=request.headers.get("authorization"),
|
||||
)
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments")
|
||||
async def upload(request: Request, chat_id: str):
|
||||
user = await actor(request)
|
||||
cid = parse_uuid(chat_id)
|
||||
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
|
||||
names = request.query_params.getlist("filename")
|
||||
if len(names) != 1:
|
||||
raise invalid()
|
||||
name = images.filename(names[0])
|
||||
declared = (
|
||||
request.headers.get("content-type", "")
|
||||
.partition(";")[0]
|
||||
.strip()
|
||||
.lower()
|
||||
)
|
||||
if declared not in {"image/png", "image/jpeg"}:
|
||||
raise DomainError(
|
||||
415, "unsupported_media_type", "PNG or JPEG required"
|
||||
)
|
||||
length = request.headers.get("content-length")
|
||||
if length and (not length.isdecimal() or len(length) > 12):
|
||||
raise invalid()
|
||||
if length and int(length) > images.MAX_BYTES:
|
||||
raise DomainError(413, "payload_too_large", "Byte limit exceeded")
|
||||
try:
|
||||
with fail_after(20):
|
||||
async with app.state.upload_slots:
|
||||
with tempfile.SpooledTemporaryFile(
|
||||
max_size=1024 * 1024, mode="w+b"
|
||||
) as stream:
|
||||
size = 0
|
||||
digest = hashlib.sha256()
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > images.MAX_BYTES:
|
||||
raise DomainError(
|
||||
413, "payload_too_large", "Byte limit exceeded"
|
||||
)
|
||||
digest.update(chunk)
|
||||
stream.write(chunk)
|
||||
await run_io(
|
||||
images.verify, stream=stream, declared=declared
|
||||
)
|
||||
result = await run_io(
|
||||
app.state.attachments.upload,
|
||||
chat_id=cid,
|
||||
owner_id=user.id,
|
||||
filename=name,
|
||||
content_type=declared,
|
||||
size=size,
|
||||
sha256=digest.hexdigest(),
|
||||
stream=stream,
|
||||
)
|
||||
return response(result, status=202)
|
||||
except ClientDisconnect:
|
||||
raise invalid() from None
|
||||
except TimeoutError:
|
||||
raise DomainError(
|
||||
503, "unavailable", "Upload deadline exceeded"
|
||||
) from None
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
async def attachment_metadata(request: Request, attachment_id: str):
|
||||
user = await actor(request)
|
||||
return response(
|
||||
await run_io(
|
||||
app.state.attachments.metadata,
|
||||
identity=parse_uuid(attachment_id),
|
||||
actor_id=user.id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
async def attachment_content(request: Request, attachment_id: str):
|
||||
user = await actor(request)
|
||||
variants = request.query_params.getlist("variant")
|
||||
if len(variants) > 1:
|
||||
raise invalid()
|
||||
row, upstream = await run_io(
|
||||
app.state.attachments.content,
|
||||
identity=parse_uuid(attachment_id),
|
||||
actor_id=user.id,
|
||||
variant=variants[0] if variants else "original",
|
||||
)
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while data := upstream.read(65536):
|
||||
yield data
|
||||
finally:
|
||||
upstream.close()
|
||||
upstream.release_conn()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=row["content_type"],
|
||||
headers={
|
||||
"Content-Length": str(row["size_bytes"]),
|
||||
"Content-Disposition": "inline; filename*=UTF-8''"
|
||||
+ quote(row["filename"], safe=""),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,144 @@
|
||||
from uuid import uuid4
|
||||
|
||||
from src.errors import DomainError, not_found
|
||||
from src.pipeline import enqueue, fault
|
||||
|
||||
|
||||
def dto(row, variants=()):
|
||||
return {
|
||||
k: row[k]
|
||||
for k in [
|
||||
"id",
|
||||
"chat_id",
|
||||
"owner_id",
|
||||
"filename",
|
||||
"content_type",
|
||||
"size_bytes",
|
||||
"sha256",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
]
|
||||
} | {
|
||||
"status": row["state"],
|
||||
"variants": [
|
||||
{
|
||||
k: v[k]
|
||||
for k in [
|
||||
"name",
|
||||
"content_type",
|
||||
"size_bytes",
|
||||
"sha256",
|
||||
"width",
|
||||
"height",
|
||||
]
|
||||
}
|
||||
for v in variants
|
||||
if v["name"] != "original"
|
||||
],
|
||||
"error_code": row["error_code"] if row["state"] == "failed" else None,
|
||||
}
|
||||
|
||||
|
||||
class Attachments:
|
||||
def __init__(self, *, database, store, instance):
|
||||
self.database, self.store, self.instance = database, store, instance
|
||||
|
||||
def upload(
|
||||
self,
|
||||
*,
|
||||
chat_id,
|
||||
owner_id,
|
||||
filename,
|
||||
content_type,
|
||||
size,
|
||||
sha256,
|
||||
stream,
|
||||
):
|
||||
identity, job_id = uuid4(), uuid4()
|
||||
key = f"objects/staging/{identity}/original"
|
||||
with self.database.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO upload_reservations(id,object_key) VALUES(%s,%s)",
|
||||
(identity, key),
|
||||
)
|
||||
self.store.put(
|
||||
key=key,
|
||||
stream=stream,
|
||||
size=size,
|
||||
content_type=content_type,
|
||||
sha256=sha256,
|
||||
)
|
||||
fault(self.database, chat_id, "api_after_staging")
|
||||
with self.database.connection() as conn, conn.transaction():
|
||||
reservation = conn.execute(
|
||||
"SELECT * FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp() FOR UPDATE",
|
||||
(identity,),
|
||||
).fetchone()
|
||||
if reservation is None:
|
||||
raise DomainError(
|
||||
503, "unavailable", "Upload reservation expired"
|
||||
)
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,uploader_instance,expires_at) "
|
||||
"VALUES(%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,clock_timestamp()) RETURNING *",
|
||||
(
|
||||
identity,
|
||||
chat_id,
|
||||
owner_id,
|
||||
filename,
|
||||
content_type,
|
||||
size,
|
||||
sha256,
|
||||
key,
|
||||
self.instance,
|
||||
),
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,1,%s,'queued')",
|
||||
(job_id, identity, key),
|
||||
)
|
||||
enqueue(conn, job_id)
|
||||
conn.execute(
|
||||
"DELETE FROM upload_reservations WHERE id=%s", (identity,)
|
||||
)
|
||||
fault(self.database, chat_id, "api_after_commit")
|
||||
# Captured committed snapshot remains queued even if worker already finished.
|
||||
return dto(row)
|
||||
|
||||
def row(self, *, identity, actor_id):
|
||||
with self.database.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id AND m.user_id=%s WHERE a.id=%s",
|
||||
(actor_id, identity),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
def metadata(self, *, identity, actor_id):
|
||||
row = self.row(identity=identity, actor_id=actor_id)
|
||||
variants = []
|
||||
if row["state"] == "ready":
|
||||
with self.database.connection() as conn:
|
||||
variants = conn.execute(
|
||||
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name",
|
||||
(identity, row["published_version"]),
|
||||
).fetchall()
|
||||
return dto(row, variants)
|
||||
|
||||
def content(self, *, identity, actor_id, variant):
|
||||
row = self.row(identity=identity, actor_id=actor_id)
|
||||
if row["state"] != "ready":
|
||||
raise DomainError(409, "conflict", "Attachment is not ready")
|
||||
with self.database.connection() as conn:
|
||||
data = conn.execute(
|
||||
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s",
|
||||
(identity, row["published_version"], variant),
|
||||
).fetchone()
|
||||
if data is None:
|
||||
raise not_found()
|
||||
if self.store.head(key=data["object_key"]) is None:
|
||||
raise DomainError(503, "unavailable", "Variant unavailable")
|
||||
return {**data, "filename": row["filename"]}, self.store.get(
|
||||
key=data["object_key"]
|
||||
)
|
||||
@@ -0,0 +1,66 @@
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
import redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
from src.domain import User, timestamp
|
||||
|
||||
|
||||
class UserCache:
|
||||
def __init__(self, *, url: str):
|
||||
self.client = redis.Redis.from_url(
|
||||
url,
|
||||
socket_connect_timeout=0.15,
|
||||
socket_timeout=0.15,
|
||||
max_connections=16,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
decode_responses=True,
|
||||
)
|
||||
self.disabled_until = 0.0
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
|
||||
def _command(self, function, *args, **kwargs):
|
||||
if time.monotonic() < self.disabled_until:
|
||||
return None
|
||||
try:
|
||||
return function(*args, **kwargs)
|
||||
except redis.RedisError, OSError:
|
||||
with self.lock:
|
||||
self.disabled_until = time.monotonic() + 5
|
||||
return None
|
||||
|
||||
def get(self, *, user_id: UUID) -> User | None:
|
||||
raw = self._command(self.client.get, "user:" + str(user_id))
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
row = json.loads(raw)
|
||||
if UUID(row["id"]) != user_id:
|
||||
return None
|
||||
return User(
|
||||
user_id,
|
||||
row["username"],
|
||||
row["display_name"],
|
||||
datetime.fromisoformat(row["created_at"]),
|
||||
)
|
||||
except ValueError, TypeError, KeyError:
|
||||
return None
|
||||
|
||||
def put(self, *, user: User):
|
||||
raw = json.dumps(
|
||||
{
|
||||
"id": str(user.id),
|
||||
"username": user.username,
|
||||
"display_name": user.display_name,
|
||||
"created_at": timestamp(user.created_at),
|
||||
}
|
||||
)
|
||||
self._command(self.client.set, "user:" + str(user.id), raw, ex=30)
|
||||
@@ -0,0 +1,64 @@
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from src.domain import Key, timestamp
|
||||
from src.errors import invalid
|
||||
|
||||
|
||||
def parse_uuid(value: str) -> UUID:
|
||||
if not isinstance(value, str) or not re.fullmatch(
|
||||
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
|
||||
):
|
||||
raise invalid()
|
||||
return UUID(value)
|
||||
|
||||
|
||||
def encode_cursor(*, scope: str, key: Key) -> str:
|
||||
payload = {
|
||||
"v": 1,
|
||||
"scope": scope,
|
||||
"at": timestamp(key[0]),
|
||||
"id": str(key[1]),
|
||||
}
|
||||
return (
|
||||
base64.urlsafe_b64encode(
|
||||
json.dumps(payload, separators=(",", ":")).encode()
|
||||
)
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
)
|
||||
|
||||
|
||||
def decode_cursor(*, value: str | None, scope: str) -> Key | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
if (
|
||||
not value
|
||||
or len(value) > 2048
|
||||
or not re.fullmatch(r"[A-Za-z0-9_-]+", value)
|
||||
):
|
||||
raise ValueError
|
||||
data = json.loads(
|
||||
base64.b64decode(
|
||||
value + "=" * (-len(value) % 4), altchars=b"-_", validate=True
|
||||
)
|
||||
)
|
||||
if (
|
||||
not isinstance(data, dict)
|
||||
or type(data.get("v")) is not int
|
||||
or data["v"] != 1
|
||||
or data.get("scope") != scope
|
||||
or not isinstance(data.get("at"), str)
|
||||
):
|
||||
raise ValueError
|
||||
at = datetime.fromisoformat(data["at"])
|
||||
if at.tzinfo is None or timestamp(at) != data["at"]:
|
||||
raise ValueError
|
||||
return at, parse_uuid(data["id"])
|
||||
except ValueError, TypeError, KeyError, binascii.Error, RecursionError:
|
||||
raise invalid() from None
|
||||
@@ -0,0 +1,77 @@
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
|
||||
from src import telemetry
|
||||
from src.errors import DomainError
|
||||
|
||||
|
||||
class DeadlineConnection(psycopg.Connection):
|
||||
"""Клиентский срок действует и при замороженном PostgreSQL."""
|
||||
|
||||
def wait(self, gen, interval=0.1, timeout=None):
|
||||
deadline = getattr(self, "request_deadline", None)
|
||||
remaining = (
|
||||
max(0.001, deadline - time.monotonic()) if deadline else 3.0
|
||||
)
|
||||
bound = min(remaining, timeout) if timeout is not None else remaining
|
||||
try:
|
||||
return super().wait(gen, interval=interval, timeout=bound)
|
||||
except psycopg.OperationalError:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, *, url: str, size: int):
|
||||
self.pool = ConnectionPool(
|
||||
url,
|
||||
connection_class=DeadlineConnection,
|
||||
kwargs={
|
||||
"autocommit": True,
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 2,
|
||||
"options": (
|
||||
"-c statement_timeout=2500 -c lock_timeout=1000 "
|
||||
"-c idle_in_transaction_session_timeout=6000"
|
||||
),
|
||||
},
|
||||
min_size=1,
|
||||
max_size=size,
|
||||
timeout=1,
|
||||
open=False,
|
||||
)
|
||||
telemetry.register_pool(self.pool)
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=10)
|
||||
|
||||
def close(self):
|
||||
self.pool.close(timeout=2)
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
with (
|
||||
telemetry.span(
|
||||
"db.postgresql",
|
||||
attributes={"db.system.name": "postgresql"},
|
||||
),
|
||||
self.pool.connection(timeout=1) as conn,
|
||||
):
|
||||
conn.request_deadline = time.monotonic() + 4
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
conn.request_deadline = None
|
||||
except psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout:
|
||||
raise DomainError(
|
||||
503, "unavailable", "Database unavailable"
|
||||
) from None
|
||||
|
||||
def ping(self):
|
||||
with self.connection() as conn:
|
||||
conn.execute("SELECT 1").fetchone()
|
||||
@@ -0,0 +1,45 @@
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def timestamp(value: datetime) -> str:
|
||||
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class User:
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Chat:
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: tuple[UUID, ...]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Message:
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: tuple[UUID, ...] = ()
|
||||
|
||||
|
||||
type Record = User | Chat | Message
|
||||
type Key = tuple[datetime, UUID]
|
||||
|
||||
|
||||
def order_key(record: Record) -> Key:
|
||||
return record.created_at, record.id
|
||||
@@ -0,0 +1,16 @@
|
||||
class DomainError(Exception):
|
||||
def __init__(
|
||||
self, status: int, code: str, message: str, headers=None
|
||||
) -> None:
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid() -> DomainError:
|
||||
return DomainError(400, "invalid_request", "Invalid request")
|
||||
|
||||
|
||||
def not_found() -> DomainError:
|
||||
return DomainError(404, "not_found", "Resource not found")
|
||||
@@ -0,0 +1,77 @@
|
||||
import struct
|
||||
import threading
|
||||
import unicodedata
|
||||
import zlib
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from src.errors import DomainError, invalid
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
DECODE_SLOT = threading.BoundedSemaphore(1)
|
||||
|
||||
|
||||
def filename(value):
|
||||
if (
|
||||
not value
|
||||
or not 1 <= len(value) <= 255
|
||||
or not value.strip()
|
||||
or any(
|
||||
c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"}
|
||||
for c in value
|
||||
)
|
||||
):
|
||||
raise invalid()
|
||||
return value
|
||||
|
||||
|
||||
def verify(stream, *, declared):
|
||||
if declared not in {"image/png", "image/jpeg"}:
|
||||
raise DomainError(
|
||||
415, "unsupported_media_type", "PNG or JPEG required"
|
||||
)
|
||||
with DECODE_SLOT:
|
||||
try:
|
||||
stream.seek(0)
|
||||
with Image.open(stream) as image:
|
||||
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(
|
||||
image.format
|
||||
)
|
||||
if actual != declared:
|
||||
raise DomainError(
|
||||
415, "unsupported_media_type", "Image type mismatch"
|
||||
)
|
||||
if image.width * image.height > MAX_PIXELS:
|
||||
raise DomainError(
|
||||
413, "payload_too_large", "Pixel limit exceeded"
|
||||
)
|
||||
image.verify()
|
||||
# JPEG verify alone does not decode entropy data.
|
||||
stream.seek(0)
|
||||
with Image.open(stream) as image:
|
||||
image.load()
|
||||
except Image.DecompressionBombError:
|
||||
raise DomainError(
|
||||
413, "payload_too_large", "Pixel limit exceeded"
|
||||
) from None
|
||||
except (
|
||||
UnidentifiedImageError,
|
||||
OSError,
|
||||
ValueError,
|
||||
SyntaxError,
|
||||
struct.error,
|
||||
zlib.error,
|
||||
EOFError,
|
||||
):
|
||||
raise DomainError(
|
||||
415, "unsupported_media_type", "Invalid image"
|
||||
) from None
|
||||
except MemoryError:
|
||||
raise DomainError(
|
||||
503, "unavailable", "Image memory budget exceeded"
|
||||
) from None
|
||||
finally:
|
||||
stream.seek(0)
|
||||
return declared
|
||||
@@ -0,0 +1,17 @@
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
message = record.getMessage()
|
||||
try:
|
||||
event = json.loads(message)
|
||||
if not isinstance(event, dict):
|
||||
event = {"event": message}
|
||||
except ValueError:
|
||||
event = {"event": message}
|
||||
event["timestamp"] = datetime.now(UTC).isoformat()
|
||||
event["level"] = record.levelname.lower()
|
||||
return json.dumps(event, ensure_ascii=False)
|
||||
@@ -0,0 +1,51 @@
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from src.settings import secret
|
||||
|
||||
|
||||
def main():
|
||||
with psycopg.connect(
|
||||
secret("DATABASE_URL"), autocommit=True, connect_timeout=3
|
||||
) as conn:
|
||||
conn.execute("SELECT pg_advisory_lock(2301002)")
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(name text PRIMARY KEY, checksum text NOT NULL, "
|
||||
"applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
for path in sorted(Path("migrations").glob("*.sql")):
|
||||
content = path.read_bytes()
|
||||
checksum = hashlib.sha256(content).hexdigest()
|
||||
row = conn.execute(
|
||||
"SELECT checksum FROM schema_migrations WHERE name=%s",
|
||||
(path.name,),
|
||||
).fetchone()
|
||||
if row:
|
||||
if row[0] != checksum:
|
||||
raise RuntimeError(
|
||||
"Applied migration checksum mismatch"
|
||||
)
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(content.decode())
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations(name,checksum) "
|
||||
"VALUES(%s,%s)",
|
||||
(path.name, checksum),
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{"event": "migration_applied", "name": path.name}
|
||||
)
|
||||
)
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(2301002)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
from contextlib import contextmanager
|
||||
|
||||
import urllib3
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
|
||||
from src import telemetry
|
||||
from src.errors import DomainError
|
||||
from src.settings import secret
|
||||
|
||||
BUCKET = "messenger"
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self):
|
||||
self.http = urllib3.PoolManager(
|
||||
maxsize=32,
|
||||
block=True,
|
||||
timeout=urllib3.Timeout(connect=1, read=3),
|
||||
retries=urllib3.Retry(total=0),
|
||||
)
|
||||
self.client = Minio(
|
||||
"s3:9000",
|
||||
access_key=secret("S3_ACCESS_KEY"),
|
||||
secret_key=secret("S3_SECRET_KEY"),
|
||||
secure=False,
|
||||
region="us-east-1",
|
||||
http_client=self.http,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def operation(self, name="request"):
|
||||
try:
|
||||
with telemetry.span("s3." + name, attributes={"operation": name}):
|
||||
yield
|
||||
except S3Error, urllib3.exceptions.HTTPError, OSError:
|
||||
raise DomainError(
|
||||
503, "unavailable", "Object store unavailable"
|
||||
) from None
|
||||
|
||||
def ping(self):
|
||||
with self.operation("head_bucket"):
|
||||
if not self.client.bucket_exists(BUCKET):
|
||||
raise DomainError(
|
||||
503, "unavailable", "Object store unavailable"
|
||||
)
|
||||
|
||||
def put(self, *, key, stream, size, content_type, sha256):
|
||||
stream.seek(0)
|
||||
with self.operation("put"):
|
||||
self.client.put_object(
|
||||
BUCKET,
|
||||
key,
|
||||
stream,
|
||||
length=size,
|
||||
content_type=content_type,
|
||||
metadata={"sha256": sha256},
|
||||
part_size=10 * 1024 * 1024,
|
||||
num_parallel_uploads=1,
|
||||
)
|
||||
|
||||
def head(self, *, key):
|
||||
with self.operation("head"):
|
||||
try:
|
||||
return self.client.stat_object(BUCKET, key)
|
||||
except S3Error as exc:
|
||||
if exc.code in {"NoSuchKey", "NoSuchObject"}:
|
||||
return None
|
||||
raise
|
||||
|
||||
def get(self, *, key):
|
||||
with self.operation("get"):
|
||||
return self.client.get_object(BUCKET, key)
|
||||
|
||||
def delete(self, *, key):
|
||||
with self.operation("delete"):
|
||||
self.client.remove_object(BUCKET, key)
|
||||
|
||||
def objects(self):
|
||||
with self.operation("list"):
|
||||
yield from self.client.list_objects(
|
||||
BUCKET, prefix="objects/", recursive=True
|
||||
)
|
||||
|
||||
def close(self):
|
||||
self.http.clear()
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Bounded DB polling; pending tasks missing the deadline count as bad SLIs."""
|
||||
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from src import telemetry
|
||||
from src.pipeline import database
|
||||
|
||||
|
||||
def poll(db, epoch):
|
||||
with db.connection() as conn, conn.transaction():
|
||||
# Evaluate unfinished jobs at their deadline, not after completion.
|
||||
# Exclude decided IDs before LIMIT to avoid starving newer jobs.
|
||||
conn.execute(
|
||||
"INSERT INTO job_slo_observations(job_id,outcome) "
|
||||
"SELECT j.id,CASE WHEN e.committed_at<=j.created_at+interval '10 seconds' "
|
||||
"THEN 'good' ELSE 'bad' END FROM jobs j "
|
||||
"LEFT JOIN effect_publications e ON e.job_id=j.id "
|
||||
"WHERE NOT EXISTS(SELECT 1 FROM job_slo_observations o WHERE o.job_id=j.id) "
|
||||
"AND j.state<>'obsolete' AND (e.committed_at IS NOT NULL "
|
||||
"OR j.state='failed' OR j.created_at<=clock_timestamp()-interval '10 seconds') "
|
||||
"ORDER BY j.created_at LIMIT 128 ON CONFLICT DO NOTHING"
|
||||
)
|
||||
queue = conn.execute(
|
||||
"SELECT count(*) FILTER(WHERE state='queued') depth,"
|
||||
"COALESCE(max(EXTRACT(epoch FROM clock_timestamp()-created_at)) "
|
||||
"FILTER(WHERE state IN ('queued','processing')),0) oldest FROM jobs"
|
||||
).fetchone()
|
||||
decisions = conn.execute(
|
||||
"SELECT outcome,count(*) n FROM job_slo_observations GROUP BY outcome"
|
||||
).fetchall()
|
||||
completed = conn.execute(
|
||||
"SELECT event,count(*) n FROM pipeline_events "
|
||||
"WHERE event IN ('effect_committed','failed') GROUP BY event"
|
||||
).fetchall()
|
||||
ready = conn.execute(
|
||||
"SELECT o.job_id,EXTRACT(epoch FROM e.committed_at-j.created_at) seconds "
|
||||
"FROM job_slo_observations o JOIN effect_publications e ON e.job_id=o.job_id "
|
||||
"JOIN jobs j ON j.id=o.job_id WHERE o.histogram_epoch IS DISTINCT FROM %s::uuid "
|
||||
"ORDER BY e.committed_at LIMIT 128 FOR UPDATE OF o SKIP LOCKED",
|
||||
(epoch,),
|
||||
).fetchall()
|
||||
for row in ready:
|
||||
conn.execute(
|
||||
"UPDATE job_slo_observations SET histogram_epoch=%s WHERE job_id=%s",
|
||||
(epoch, row["job_id"]),
|
||||
)
|
||||
values = {
|
||||
"queue_depth": queue["depth"],
|
||||
"oldest_job_age": float(queue["oldest"]),
|
||||
"slo_good": 0,
|
||||
"slo_bad": 0,
|
||||
"completed_ready": 0,
|
||||
"completed_failed": 0,
|
||||
}
|
||||
values.update({"slo_" + row["outcome"]: row["n"] for row in decisions})
|
||||
values.update(
|
||||
{
|
||||
"completed_"
|
||||
+ (
|
||||
"ready" if row["event"] == "effect_committed" else "failed"
|
||||
): row["n"]
|
||||
for row in completed
|
||||
}
|
||||
)
|
||||
telemetry.observe(values)
|
||||
for row in ready:
|
||||
telemetry.duration(
|
||||
"time_to_ready",
|
||||
float(row["seconds"]),
|
||||
{"job_type": "attachment.process.v1", "outcome": "ready"},
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
epoch = str(uuid4())
|
||||
telemetry.configure("messenger-observer", instance=epoch)
|
||||
db = database()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
poll(db, epoch)
|
||||
except Exception:
|
||||
telemetry.log(
|
||||
"observer.unavailable",
|
||||
error=True,
|
||||
error_code="database_unavailable",
|
||||
)
|
||||
time.sleep(2)
|
||||
finally:
|
||||
db.close()
|
||||
telemetry.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
"""Explicit local operator actions. There is deliberately no HTTP replay endpoint."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from src.object_store import ObjectStore
|
||||
from src.pipeline import database, enqueue, event
|
||||
|
||||
|
||||
def change(db, store, *, job_id, operator, reason, version=None):
|
||||
if (
|
||||
not operator.strip()
|
||||
or not reason.strip()
|
||||
or len(operator) > 100
|
||||
or len(reason) > 500
|
||||
):
|
||||
raise ValueError("Operator and reason are required")
|
||||
with db.connection() as conn, conn.transaction():
|
||||
job = conn.execute(
|
||||
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
|
||||
).fetchone()
|
||||
if not job:
|
||||
raise ValueError("Unknown job")
|
||||
attachment = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE",
|
||||
(job["attachment_id"],),
|
||||
).fetchone()
|
||||
if job["pipeline_version"] != attachment["pipeline_version"]:
|
||||
raise ValueError("Superseded job cannot be replayed")
|
||||
source = (
|
||||
job["source_key"]
|
||||
if attachment["published_version"] is None
|
||||
else attachment["object_key"]
|
||||
)
|
||||
if store.head(key=source) is None:
|
||||
raise ValueError("Source expired or missing; replay refused")
|
||||
if version is None:
|
||||
if job["state"] != "failed":
|
||||
raise ValueError("Only failed jobs can be replayed")
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state='queued',attempt=0,cycle=cycle+1,fence=fence+1,error_code=NULL,next_at=clock_timestamp(),lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(job_id,),
|
||||
)
|
||||
action = "replay"
|
||||
target = job_id
|
||||
else:
|
||||
if (
|
||||
version not in {1, 2}
|
||||
or version <= attachment["pipeline_version"]
|
||||
):
|
||||
raise ValueError("Supported version must increase (1 or 2)")
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state='obsolete',fence=fence+1,lease_owner=NULL,lease_until=NULL WHERE id=%s",
|
||||
(job_id,),
|
||||
)
|
||||
target = uuid4()
|
||||
conn.execute(
|
||||
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,%s,%s,'queued')",
|
||||
(target, attachment["id"], version, source),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET pipeline_version=%s WHERE id=%s",
|
||||
(version, attachment["id"]),
|
||||
)
|
||||
action = "new_version"
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(attachment["id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_audit(job_id,action,operator,reason) VALUES(%s,%s,%s,%s)",
|
||||
(target, action, operator, reason),
|
||||
)
|
||||
event(conn, target, action, {"previous_job": str(job_id)})
|
||||
enqueue(conn, target)
|
||||
return str(target)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("action", choices=["failed", "replay", "version"])
|
||||
p.add_argument("--job-id", type=UUID)
|
||||
p.add_argument("--operator")
|
||||
p.add_argument("--reason")
|
||||
p.add_argument("--version", type=int)
|
||||
a = p.parse_args()
|
||||
db, store = database(), ObjectStore()
|
||||
try:
|
||||
if a.action == "failed":
|
||||
with db.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id,attachment_id,pipeline_version,attempt,cycle,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at,id"
|
||||
).fetchall()
|
||||
print(json.dumps(rows, default=str))
|
||||
else:
|
||||
if not all([a.job_id, a.operator, a.reason]):
|
||||
p.error("job-id/operator/reason required")
|
||||
if a.action == "version" and a.version is None:
|
||||
p.error("version required")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"job_id": change(
|
||||
db,
|
||||
store,
|
||||
job_id=a.job_id,
|
||||
operator=a.operator,
|
||||
reason=a.reason,
|
||||
version=a.version
|
||||
if a.action == "version"
|
||||
else None,
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,36 @@
|
||||
import threading
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
from argon2.low_level import Type
|
||||
|
||||
from src.errors import DomainError
|
||||
|
||||
|
||||
class Passwords:
|
||||
def __init__(self):
|
||||
self.hasher = PasswordHasher(
|
||||
time_cost=3, memory_cost=65536, parallelism=4, type=Type.ID
|
||||
)
|
||||
self.slots = threading.BoundedSemaphore(2)
|
||||
self.dummy = self.hasher.hash("Synthetic-dummy-password-not-for-login")
|
||||
|
||||
def _run(self, function, *args):
|
||||
if not self.slots.acquire(timeout=5):
|
||||
raise DomainError(503, "unavailable", "Authentication busy")
|
||||
try:
|
||||
return function(*args)
|
||||
finally:
|
||||
self.slots.release()
|
||||
|
||||
def hash(self, *, password: str) -> str:
|
||||
return self._run(self.hasher.hash, password)
|
||||
|
||||
def verify(self, *, password: str, encoded: str | None) -> bool:
|
||||
try:
|
||||
return (
|
||||
self._run(self.hasher.verify, encoded or self.dummy, password)
|
||||
and encoded is not None
|
||||
)
|
||||
except VerificationError, InvalidHashError:
|
||||
return False
|
||||
+289
@@ -0,0 +1,289 @@
|
||||
"""Durable job state is authoritative; stream entries are wake-up notifications."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
import redis
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from src import telemetry
|
||||
from src.database import Database
|
||||
from src.settings import integer, secret
|
||||
|
||||
STREAM = "attachments"
|
||||
GROUP = "image-workers"
|
||||
|
||||
|
||||
def database():
|
||||
db = Database(url=secret("DATABASE_URL"), size=4)
|
||||
db.open()
|
||||
return db
|
||||
|
||||
|
||||
def broker():
|
||||
return redis.Redis.from_url(
|
||||
secret("BROKER_URL"),
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=1,
|
||||
socket_timeout=3,
|
||||
max_connections=8,
|
||||
)
|
||||
|
||||
|
||||
def envelope(row):
|
||||
return dict(
|
||||
schema_version=1,
|
||||
job_id=str(row["id"]),
|
||||
type="attachment.process.v1",
|
||||
attachment_id=str(row["attachment_id"]),
|
||||
pipeline_version=row["pipeline_version"],
|
||||
attempt=row["attempt"] + 1,
|
||||
created_at=datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||
)
|
||||
|
||||
|
||||
def validate_job(data):
|
||||
required = {
|
||||
"schema_version",
|
||||
"job_id",
|
||||
"type",
|
||||
"attachment_id",
|
||||
"pipeline_version",
|
||||
"attempt",
|
||||
"created_at",
|
||||
}
|
||||
if (
|
||||
not isinstance(data, dict)
|
||||
or not required <= data.keys()
|
||||
or data.keys() - required - {"traceparent", "tracestate"}
|
||||
):
|
||||
raise ValueError("Invalid job envelope")
|
||||
if (
|
||||
type(data["schema_version"]) is not int
|
||||
or data["schema_version"] != 1
|
||||
or data["type"] != "attachment.process.v1"
|
||||
):
|
||||
raise ValueError("Invalid job type")
|
||||
for k in ["job_id", "attachment_id"]:
|
||||
if not isinstance(data[k], str):
|
||||
raise ValueError("Invalid job UUID")
|
||||
UUID(data[k])
|
||||
for k in ["pipeline_version", "attempt"]:
|
||||
if type(data[k]) is not int or data[k] < 1:
|
||||
raise ValueError("Invalid job number")
|
||||
timestamp = data["created_at"]
|
||||
if not isinstance(timestamp, str) or not re.fullmatch(
|
||||
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})",
|
||||
timestamp,
|
||||
):
|
||||
raise ValueError("Invalid job timestamp")
|
||||
datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
if "traceparent" in data:
|
||||
if not isinstance(data["traceparent"], str) or not re.fullmatch(
|
||||
r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"]
|
||||
):
|
||||
raise ValueError("Invalid trace context")
|
||||
if "tracestate" in data and (
|
||||
not isinstance(data["tracestate"], str)
|
||||
or len(data["tracestate"]) > 512
|
||||
):
|
||||
raise ValueError("Invalid trace context")
|
||||
return data
|
||||
|
||||
|
||||
def event(conn, job_id, name, detail=None):
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_events(job_id,event,detail) VALUES(%s,%s,%s)",
|
||||
(job_id, name, Jsonb(detail or {})),
|
||||
)
|
||||
|
||||
|
||||
def enqueue(conn, job_id):
|
||||
row = conn.execute("SELECT * FROM jobs WHERE id=%s", (job_id,)).fetchone()
|
||||
context = telemetry.carrier()
|
||||
if not context and row.get("traceparent"):
|
||||
context = {"traceparent": row["traceparent"]}
|
||||
if row.get("tracestate"):
|
||||
context["tracestate"] = row["tracestate"]
|
||||
if context:
|
||||
conn.execute(
|
||||
"UPDATE jobs SET traceparent=%s,tracestate=%s WHERE id=%s",
|
||||
(
|
||||
context.get("traceparent", ""),
|
||||
context.get("tracestate", ""),
|
||||
job_id,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO outbox(job_id,payload,due_at) VALUES(%s,%s,%s)",
|
||||
(job_id, Jsonb(envelope(row) | context), row["next_at"]),
|
||||
)
|
||||
|
||||
|
||||
def fault(db, target, phase):
|
||||
if os.getenv("PIPELINE_FAULTS") != "1":
|
||||
return
|
||||
with db.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT * FROM pipeline_faults WHERE target=%s AND phase=%s AND remaining>0 ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1",
|
||||
(target, phase),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return
|
||||
conn.execute(
|
||||
"UPDATE pipeline_faults SET remaining=remaining-1,hit_at=clock_timestamp() WHERE id=%s",
|
||||
(row["id"],),
|
||||
)
|
||||
action = row["action"]
|
||||
if action == "crash":
|
||||
os._exit(89)
|
||||
if action == "permanent":
|
||||
raise PermanentFailure("invalid_image")
|
||||
if action == "transient":
|
||||
raise RuntimeError("injected_transient")
|
||||
if action == "hang":
|
||||
time.sleep(120)
|
||||
if action == "wait":
|
||||
end = time.monotonic() + 60
|
||||
while time.monotonic() < end:
|
||||
with db.connection() as conn:
|
||||
released = conn.execute(
|
||||
"SELECT released_at FROM pipeline_faults WHERE id=%s",
|
||||
(row["id"],),
|
||||
).fetchone()
|
||||
if released and released["released_at"]:
|
||||
return
|
||||
time.sleep(0.05)
|
||||
raise RuntimeError("fault_wait_timeout")
|
||||
|
||||
|
||||
class PermanentFailure(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def retry(conn, row, error, *, permanent=False):
|
||||
maximum = integer("JOB_MAX_ATTEMPTS", 5, 1, 20)
|
||||
failed = permanent or row["attempt"] >= maximum
|
||||
delay = random.uniform(0.25, min(30, 2 ** row["attempt"]))
|
||||
state = "failed" if failed else "queued"
|
||||
code = error if permanent else ("retry_exhausted" if failed else error)
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state=%s,fence=fence+1,lease_owner=NULL,lease_until=NULL,error_code=%s,next_at=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s",
|
||||
(state, code, delay, row["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
|
||||
(
|
||||
state,
|
||||
code if failed else None,
|
||||
row["attachment_id"],
|
||||
row["pipeline_version"],
|
||||
),
|
||||
)
|
||||
event(
|
||||
conn,
|
||||
row["id"],
|
||||
"failed" if failed else "retry",
|
||||
{"attempt": row["attempt"], "code": code, "backoff_seconds": delay},
|
||||
)
|
||||
if not failed:
|
||||
enqueue(conn, row["id"])
|
||||
|
||||
|
||||
def watchdog(db):
|
||||
with db.connection() as conn, conn.transaction():
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM jobs WHERE state='processing' AND lease_until<clock_timestamp() ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 16"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
retry(conn, row, "lease_expired")
|
||||
# Recreate a wakeup if its stream entry was lost: never recreate logical jobs.
|
||||
rows = conn.execute(
|
||||
"SELECT j.* FROM jobs j WHERE j.state='queued' AND j.next_at<=clock_timestamp() AND NOT EXISTS(SELECT 1 FROM outbox o WHERE o.job_id=j.id AND (o.published_at IS NULL OR o.published_at>clock_timestamp()-interval '15 seconds')) FOR UPDATE OF j SKIP LOCKED LIMIT 16"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
enqueue(conn, row["id"])
|
||||
|
||||
|
||||
def relay_once(db, queue):
|
||||
watchdog(db)
|
||||
with db.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT * FROM outbox WHERE published_at IS NULL AND due_at<=clock_timestamp() ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1"
|
||||
).fetchone()
|
||||
if not row:
|
||||
return False
|
||||
validate_job(row["payload"])
|
||||
with telemetry.span(
|
||||
"attachments publish",
|
||||
kind=telemetry.SpanKind.PRODUCER,
|
||||
parent=row["payload"],
|
||||
attributes={
|
||||
"job.id": str(row["job_id"]),
|
||||
"messaging.system": "valkey",
|
||||
"messaging.destination.name": STREAM,
|
||||
},
|
||||
):
|
||||
# WAITAOF must use the SAME connection as XADD (not two pooled calls).
|
||||
with queue.client() as dedicated:
|
||||
entry = dedicated.xadd(
|
||||
STREAM,
|
||||
{
|
||||
"job": json.dumps(
|
||||
row["payload"] | telemetry.carrier(),
|
||||
separators=(",", ":"),
|
||||
)
|
||||
},
|
||||
)
|
||||
confirmation = dedicated.execute_command("WAITAOF", 1, 0, 2000)
|
||||
if int(confirmation[0]) < 1:
|
||||
raise RuntimeError(
|
||||
"Broker durability confirmation timed out"
|
||||
)
|
||||
telemetry.log("job.published", job_id=str(row["job_id"]))
|
||||
fault(db, row["job_id"], "relay_after_publish")
|
||||
conn.execute(
|
||||
"UPDATE outbox SET published_at=clock_timestamp() WHERE id=%s",
|
||||
(row["id"],),
|
||||
)
|
||||
event(
|
||||
conn,
|
||||
row["job_id"],
|
||||
"published",
|
||||
{"entry": entry, "outbox": row["id"]},
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def relay():
|
||||
telemetry.configure("messenger-relay")
|
||||
db, queue = database(), broker()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
if not relay_once(db, queue):
|
||||
time.sleep(0.2)
|
||||
except Exception:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "relay_retry",
|
||||
"code": "dependency_unavailable",
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(1)
|
||||
finally:
|
||||
queue.close()
|
||||
db.close()
|
||||
telemetry.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
relay()
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Retention cleanup; default is dry-run. Active intent always protects bytes."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
|
||||
from src.object_store import ObjectStore
|
||||
from src.pipeline import database
|
||||
from src.settings import integer
|
||||
|
||||
|
||||
def reconcile(db, store, *, apply=False, grace=None):
|
||||
grace = (
|
||||
grace
|
||||
if grace is not None
|
||||
else integer("STAGING_RETENTION_SECONDS", 86400, 300, 604800)
|
||||
)
|
||||
if grace < 300:
|
||||
raise ValueError("Retention must be at least 300 seconds")
|
||||
actions = []
|
||||
for obj in store.objects():
|
||||
# Recheck each candidate in a bounded transaction, including late PUT age.
|
||||
with db.connection() as conn, conn.transaction():
|
||||
now = conn.execute("SELECT clock_timestamp() now").fetchone()[
|
||||
"now"
|
||||
]
|
||||
if (now - obj.last_modified).total_seconds() < grace:
|
||||
continue
|
||||
reservation = conn.execute(
|
||||
"SELECT * FROM upload_reservations WHERE object_key=%s FOR UPDATE",
|
||||
(obj.object_name,),
|
||||
).fetchone()
|
||||
if reservation and reservation["expires_at"] > now:
|
||||
continue
|
||||
jobs = conn.execute(
|
||||
"SELECT * FROM jobs WHERE source_key=%s OR %s LIKE 'objects/' || attachment_id::text || '/v' || pipeline_version::text || '/%%' FOR UPDATE",
|
||||
(obj.object_name, obj.object_name),
|
||||
).fetchall()
|
||||
if any(j["state"] in {"queued", "processing"} for j in jobs):
|
||||
continue
|
||||
current = conn.execute(
|
||||
"SELECT v.* FROM attachment_variants v JOIN attachments a ON a.id=v.attachment_id AND a.published_version=v.pipeline_version WHERE v.object_key=%s",
|
||||
(obj.object_name,),
|
||||
).fetchone()
|
||||
if current:
|
||||
continue
|
||||
if any(
|
||||
j["state"] == "failed"
|
||||
and (now - j["updated_at"]).total_seconds() < grace
|
||||
for j in jobs
|
||||
):
|
||||
continue
|
||||
head = store.head(key=obj.object_name)
|
||||
if (
|
||||
head is None
|
||||
or (now - head.last_modified).total_seconds() < grace
|
||||
):
|
||||
continue
|
||||
action = {
|
||||
"action": "delete_expired_object",
|
||||
"key": obj.object_name,
|
||||
"applied": apply,
|
||||
}
|
||||
if apply:
|
||||
# Taking the reservation lock fences an expired API before deletion.
|
||||
if reservation:
|
||||
conn.execute(
|
||||
"DELETE FROM upload_reservations WHERE id=%s",
|
||||
(reservation["id"],),
|
||||
)
|
||||
store.delete(key=obj.object_name)
|
||||
for job in jobs:
|
||||
if (
|
||||
job["state"] == "failed"
|
||||
and job["source_key"] == obj.object_name
|
||||
):
|
||||
conn.execute(
|
||||
"UPDATE jobs SET error_code='source_expired' WHERE id=%s",
|
||||
(job["id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET error_code='source_expired' WHERE id=%s AND pipeline_version=%s AND state='failed'",
|
||||
(job["attachment_id"], job["pipeline_version"]),
|
||||
)
|
||||
actions.append(action)
|
||||
return {"apply": apply, "retention_seconds": grace, "actions": actions}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--apply", action="store_true")
|
||||
p.add_argument("--loop", action="store_true")
|
||||
a = p.parse_args()
|
||||
db, store = database(), ObjectStore()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
print(
|
||||
json.dumps(reconcile(db, store, apply=a.apply)), flush=True
|
||||
)
|
||||
except Exception:
|
||||
if not a.loop:
|
||||
raise
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "cleanup_retry",
|
||||
"code": "dependency_unavailable",
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
if not a.loop:
|
||||
break
|
||||
time.sleep(60)
|
||||
finally:
|
||||
db.close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,296 @@
|
||||
import hashlib
|
||||
import json
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg.errors import ForeignKeyViolation, UniqueViolation
|
||||
|
||||
from src.cache import UserCache
|
||||
from src.database import Database
|
||||
from src.domain import Chat, Key, Message, User
|
||||
from src.errors import DomainError, invalid, not_found
|
||||
|
||||
MESSAGE_COLUMNS = (
|
||||
"m.*, ARRAY(SELECT attachment_id FROM message_attachments ma "
|
||||
"WHERE ma.message_id=m.id ORDER BY ma.position) attachment_ids"
|
||||
)
|
||||
|
||||
USER_COLUMNS = "id, username, display_name, created_at"
|
||||
|
||||
|
||||
def user_dto(row) -> User:
|
||||
return User(
|
||||
row["id"], row["username"], row["display_name"], row["created_at"]
|
||||
)
|
||||
|
||||
|
||||
def chat_dto(row) -> Chat:
|
||||
return Chat(
|
||||
row["id"], row["title"], tuple(row["member_ids"]), row["created_at"]
|
||||
)
|
||||
|
||||
|
||||
def message_dto(row) -> Message:
|
||||
return Message(
|
||||
row["id"],
|
||||
row["chat_id"],
|
||||
row["sender_id"],
|
||||
row["text"],
|
||||
row["created_at"],
|
||||
tuple(row.get("attachment_ids", ())),
|
||||
)
|
||||
|
||||
|
||||
class PostgresRepository:
|
||||
def __init__(self, *, database: Database, cache: UserCache):
|
||||
self.database = database
|
||||
self.cache = cache
|
||||
|
||||
def add_user(self, *, user: User, password_hash: str) -> User:
|
||||
try:
|
||||
with self.database.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO users(id,username,display_name,"
|
||||
"password_hash,created_at) VALUES(%s,%s,%s,%s,%s)",
|
||||
(
|
||||
user.id,
|
||||
user.username,
|
||||
user.display_name,
|
||||
password_hash,
|
||||
user.created_at,
|
||||
),
|
||||
)
|
||||
except UniqueViolation:
|
||||
raise DomainError(
|
||||
409, "conflict", "Username already exists"
|
||||
) from None
|
||||
return user
|
||||
|
||||
def get_user(self, *, user_id: UUID) -> User | None:
|
||||
cached = self.cache.get(user_id=user_id)
|
||||
if cached is not None:
|
||||
return cached
|
||||
with self.database.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT " + USER_COLUMNS + " FROM users WHERE id=%s",
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
user = user_dto(row)
|
||||
self.cache.put(user=user)
|
||||
return user
|
||||
|
||||
def credentials(self, *, username: str):
|
||||
with self.database.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT "
|
||||
+ USER_COLUMNS
|
||||
+ ", password_hash FROM users WHERE username=%s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None, None
|
||||
return user_dto(row), row["password_hash"]
|
||||
|
||||
def username_exists(self, *, username: str) -> bool:
|
||||
with self.database.connection() as conn:
|
||||
return (
|
||||
conn.execute(
|
||||
"SELECT 1 FROM users WHERE username=%s", (username,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
|
||||
def add_session(
|
||||
self, *, digest: bytes, user_id: UUID, created_at, expires_at
|
||||
):
|
||||
with self.database.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(token_digest,user_id,created_at,"
|
||||
"expires_at) VALUES(%s,%s,%s,%s)",
|
||||
(digest, user_id, created_at, expires_at),
|
||||
)
|
||||
|
||||
def session_user(self, *, digest: bytes) -> UUID | None:
|
||||
with self.database.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT user_id FROM sessions WHERE token_digest=%s "
|
||||
"AND expires_at>clock_timestamp()",
|
||||
(digest,),
|
||||
).fetchone()
|
||||
return row["user_id"] if row else None
|
||||
|
||||
def delete_session(self, *, digest: bytes):
|
||||
with self.database.connection() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM sessions WHERE token_digest=%s", (digest,)
|
||||
)
|
||||
|
||||
def add_chat(self, *, chat: Chat, actor_id: UUID) -> Chat:
|
||||
members = chat.member_ids
|
||||
if actor_id not in members or len(set(members)) != len(members):
|
||||
raise invalid()
|
||||
try:
|
||||
with self.database.connection() as conn, conn.transaction():
|
||||
conn.execute(
|
||||
"INSERT INTO chats(id,title,created_at) VALUES(%s,%s,%s)",
|
||||
(chat.id, chat.title, chat.created_at),
|
||||
)
|
||||
with conn.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"INSERT INTO members(chat_id,user_id,position) "
|
||||
"VALUES(%s,%s,%s)",
|
||||
[
|
||||
(chat.id, user_id, position)
|
||||
for position, user_id in enumerate(members)
|
||||
],
|
||||
)
|
||||
except ForeignKeyViolation:
|
||||
raise invalid() from None
|
||||
return chat
|
||||
|
||||
def get_chat(self, *, chat_id: UUID, actor_id: UUID) -> Chat:
|
||||
with self.database.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT c.id,c.title,c.created_at,"
|
||||
"array_agg(m.user_id ORDER BY m.position) member_ids "
|
||||
"FROM chats c JOIN members m ON m.chat_id=c.id "
|
||||
"WHERE c.id=%s AND EXISTS(SELECT 1 FROM members own "
|
||||
"WHERE own.chat_id=c.id AND own.user_id=%s) GROUP BY c.id",
|
||||
(chat_id, actor_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise not_found()
|
||||
return chat_dto(row)
|
||||
|
||||
def add_message(
|
||||
self, *, message: Message, idempotency_key: str
|
||||
) -> Message:
|
||||
# Область включает пользователя, метод и нормализованный путь чата.
|
||||
scope = f"{message.sender_id}:POST:{message.chat_id}:{idempotency_key}"
|
||||
lock = int.from_bytes(
|
||||
hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True
|
||||
)
|
||||
fingerprint = hashlib.sha256(
|
||||
json.dumps(
|
||||
{
|
||||
"text": message.text,
|
||||
"attachment_ids": [str(x) for x in message.attachment_ids],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
).digest()
|
||||
try:
|
||||
with self.database.connection() as conn, conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
prior = conn.execute(
|
||||
"SELECT i.body_hash,i.message_id,"
|
||||
"i.expires_at>clock_timestamp() active "
|
||||
"FROM idempotency i WHERE user_id=%s AND chat_id=%s "
|
||||
"AND key=%s FOR UPDATE",
|
||||
(message.sender_id, message.chat_id, idempotency_key),
|
||||
).fetchone()
|
||||
if prior and prior["active"]:
|
||||
if bytes(prior["body_hash"]) != fingerprint:
|
||||
raise DomainError(409, "conflict", "Key already used")
|
||||
row = conn.execute(
|
||||
"SELECT "
|
||||
+ MESSAGE_COLUMNS
|
||||
+ " FROM messages m WHERE m.id=%s",
|
||||
(prior["message_id"],),
|
||||
).fetchone()
|
||||
return message_dto(row)
|
||||
if message.attachment_ids:
|
||||
rows = conn.execute(
|
||||
"SELECT id,state FROM attachments WHERE chat_id=%s "
|
||||
"AND id=ANY(%s) FOR SHARE",
|
||||
(message.chat_id, list(message.attachment_ids)),
|
||||
).fetchall()
|
||||
if len(rows) != len(message.attachment_ids):
|
||||
raise not_found()
|
||||
if any(row["state"] == "failed" for row in rows):
|
||||
raise DomainError(
|
||||
409, "conflict", "Attachment is not ready"
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO messages(id,chat_id,sender_id,"
|
||||
"text,created_at) "
|
||||
"VALUES(%s,%s,%s,%s,%s)",
|
||||
(
|
||||
message.id,
|
||||
message.chat_id,
|
||||
message.sender_id,
|
||||
message.text,
|
||||
message.created_at,
|
||||
),
|
||||
)
|
||||
if message.attachment_ids:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"INSERT INTO message_attachments("
|
||||
"message_id,chat_id,"
|
||||
"attachment_id,position) "
|
||||
"VALUES(%s,%s,%s,%s)",
|
||||
[
|
||||
(message.id, message.chat_id, aid, i)
|
||||
for i, aid in enumerate(message.attachment_ids)
|
||||
],
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO idempotency(user_id,chat_id,key,body_hash,"
|
||||
"message_id,expires_at) VALUES(%s,%s,%s,%s,%s,"
|
||||
"clock_timestamp()+interval '24 hours') "
|
||||
"ON CONFLICT(user_id,chat_id,key) DO UPDATE SET "
|
||||
"body_hash=excluded.body_hash,message_id=excluded.message_id,"
|
||||
"expires_at=excluded.expires_at",
|
||||
(
|
||||
message.sender_id,
|
||||
message.chat_id,
|
||||
idempotency_key,
|
||||
fingerprint,
|
||||
message.id,
|
||||
),
|
||||
)
|
||||
except ForeignKeyViolation:
|
||||
raise not_found() from None
|
||||
return message
|
||||
|
||||
def list_chats(
|
||||
self, *, actor_id: UUID, after: Key | None, limit: int
|
||||
) -> list[Chat]:
|
||||
condition = "AND (c.created_at,c.id)>(%s,%s)" if after else ""
|
||||
parameters = (
|
||||
[actor_id, *after, limit + 1] if after else [actor_id, limit + 1]
|
||||
)
|
||||
with self.database.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id "
|
||||
"FROM members WHERE chat_id=c.id "
|
||||
"ORDER BY position) member_ids "
|
||||
"FROM chats c JOIN members own ON own.chat_id=c.id "
|
||||
"WHERE own.user_id=%s "
|
||||
+ condition
|
||||
+ " ORDER BY c.created_at,c.id LIMIT %s",
|
||||
parameters,
|
||||
).fetchall()
|
||||
return [chat_dto(row) for row in rows]
|
||||
|
||||
def list_messages(
|
||||
self, *, chat_id: UUID, actor_id: UUID, after: Key | None, limit: int
|
||||
) -> list[Message]:
|
||||
condition = "AND (m.created_at,m.id)>(%s,%s)" if after else ""
|
||||
parameters = (
|
||||
[chat_id, *after, limit + 1] if after else [chat_id, limit + 1]
|
||||
)
|
||||
with self.database.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT "
|
||||
+ MESSAGE_COLUMNS
|
||||
+ " FROM messages m WHERE m.chat_id=%s "
|
||||
+ condition
|
||||
+ " ORDER BY m.created_at,m.id LIMIT %s",
|
||||
parameters,
|
||||
).fetchall()
|
||||
return [message_dto(row) for row in rows]
|
||||
@@ -0,0 +1,65 @@
|
||||
import re
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
BeforeValidator,
|
||||
ConfigDict,
|
||||
Field,
|
||||
StrictStr,
|
||||
ValidationInfo,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from src.cursor import parse_uuid
|
||||
|
||||
UuidIn = Annotated[UUID, BeforeValidator(parse_uuid)]
|
||||
|
||||
|
||||
class Input(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@field_validator("*", mode="after")
|
||||
@classmethod
|
||||
def nonblank(cls, value: object, info: ValidationInfo) -> object:
|
||||
if isinstance(value, str) and (
|
||||
(not value.strip() and info.field_name != "text")
|
||||
or any(0xD800 <= ord(c) <= 0xDFFF for c in value)
|
||||
):
|
||||
raise ValueError("Invalid text")
|
||||
return value
|
||||
|
||||
|
||||
class CreateUser(Input):
|
||||
username: StrictStr
|
||||
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||
password: StrictStr = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def username_format(cls, value: str) -> str:
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
|
||||
raise ValueError("Invalid username")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(Input):
|
||||
title: StrictStr = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UuidIn] = Field(min_length=1, max_length=100, strict=True)
|
||||
|
||||
|
||||
class CreateMessage(Input):
|
||||
text: StrictStr = Field(max_length=4000)
|
||||
attachment_ids: list[UuidIn] = Field(
|
||||
default_factory=list, max_length=10, strict=True
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def content(self):
|
||||
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
||||
raise ValueError("Duplicate attachment")
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError("Empty message")
|
||||
return self
|
||||
+192
@@ -0,0 +1,192 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import threading
|
||||
from datetime import timedelta
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from src.cursor import decode_cursor, encode_cursor
|
||||
from src.domain import Chat, Message, Record, User, order_key, utcnow
|
||||
from src.errors import DomainError, invalid, not_found
|
||||
from src.password import Passwords
|
||||
from src.repository import PostgresRepository
|
||||
|
||||
|
||||
def page(*, rows: list[Record], limit: int, scope: str) -> dict:
|
||||
items = rows[:limit]
|
||||
cursor = (
|
||||
encode_cursor(scope=scope, key=order_key(items[-1]))
|
||||
if len(rows) > limit
|
||||
else None
|
||||
)
|
||||
return {"items": items, "next_cursor": cursor}
|
||||
|
||||
|
||||
def unauthorized(*, basic=False):
|
||||
challenge = (
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
if basic
|
||||
else 'Bearer realm="messenger"'
|
||||
)
|
||||
return DomainError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": challenge},
|
||||
)
|
||||
|
||||
|
||||
def bearer(value):
|
||||
if not value:
|
||||
raise unauthorized()
|
||||
scheme, _, token = value.partition(" ")
|
||||
if scheme.lower() != "bearer" or not re.fullmatch(
|
||||
r"[A-Za-z0-9_-]{32,256}", token
|
||||
):
|
||||
raise unauthorized()
|
||||
return token
|
||||
|
||||
|
||||
def token_digest(value):
|
||||
return hashlib.sha256(value.encode()).digest()
|
||||
|
||||
|
||||
class MessengerService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
repository: PostgresRepository,
|
||||
passwords: Passwords,
|
||||
session_ttl: int,
|
||||
) -> None:
|
||||
self.repository = repository
|
||||
self.passwords = passwords
|
||||
self.session_ttl = session_ttl
|
||||
self.registration_locks = [threading.Lock() for _ in range(64)]
|
||||
|
||||
def identify(self, *, value: str | None) -> User:
|
||||
token = bearer(value)
|
||||
user_id = self.repository.session_user(digest=token_digest(token))
|
||||
user = self.repository.get_user(user_id=user_id) if user_id else None
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
return user
|
||||
|
||||
def register(
|
||||
self, *, username: str, display_name: str, password: str
|
||||
) -> User:
|
||||
# Bound lock memory and avoid running Argon2 for every local duplicate.
|
||||
# Different replicas still race safely through PostgreSQL UNIQUE.
|
||||
stripe = hashlib.sha256(username.encode()).digest()[0] % 64
|
||||
lock = self.registration_locks[stripe]
|
||||
if not lock.acquire(timeout=8):
|
||||
raise DomainError(503, "unavailable", "Registration busy")
|
||||
try:
|
||||
if self.repository.username_exists(username=username):
|
||||
raise DomainError(409, "conflict", "Username already exists")
|
||||
encoded = self.passwords.hash(password=password)
|
||||
return self.repository.add_user(
|
||||
user=User(uuid4(), username, display_name, utcnow()),
|
||||
password_hash=encoded,
|
||||
)
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
def login(self, *, authorization: str | None):
|
||||
try:
|
||||
scheme, _, encoded = (authorization or "").partition(" ")
|
||||
if scheme.lower() != "basic":
|
||||
raise ValueError
|
||||
plain = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, separator, password = plain.partition(":")
|
||||
if (
|
||||
not separator
|
||||
or not re.fullmatch(r"[a-z0-9_]{3,32}", username)
|
||||
or not 12 <= len(password) <= 128
|
||||
or not password.strip()
|
||||
):
|
||||
raise ValueError
|
||||
except ValueError, UnicodeError:
|
||||
raise unauthorized(basic=True) from None
|
||||
user, password_hash = self.repository.credentials(username=username)
|
||||
if not self.passwords.verify(password=password, encoded=password_hash):
|
||||
raise unauthorized(basic=True)
|
||||
now = utcnow()
|
||||
expires = now + timedelta(seconds=self.session_ttl)
|
||||
token = secrets.token_urlsafe(32)
|
||||
self.repository.add_session(
|
||||
digest=token_digest(token),
|
||||
user_id=user.id,
|
||||
created_at=now,
|
||||
expires_at=expires,
|
||||
)
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": expires,
|
||||
"user": user,
|
||||
}
|
||||
|
||||
def logout(self, *, authorization: str | None):
|
||||
self.identify(value=authorization)
|
||||
self.repository.delete_session(
|
||||
digest=token_digest(bearer(authorization))
|
||||
)
|
||||
|
||||
def create_chat(
|
||||
self, *, actor_id: UUID, title: str, member_ids: list[UUID]
|
||||
) -> Chat:
|
||||
return self.repository.add_chat(
|
||||
chat=Chat(uuid4(), title, tuple(member_ids), utcnow()),
|
||||
actor_id=actor_id,
|
||||
)
|
||||
|
||||
def get_chat(self, *, actor_id: UUID, chat_id: UUID) -> Chat:
|
||||
return self.repository.get_chat(actor_id=actor_id, chat_id=chat_id)
|
||||
|
||||
def send(
|
||||
self,
|
||||
*,
|
||||
actor_id: UUID,
|
||||
chat_id: UUID,
|
||||
text: str,
|
||||
idempotency_key: str | None,
|
||||
attachment_ids: list[UUID],
|
||||
) -> Message:
|
||||
if not idempotency_key or not re.fullmatch(
|
||||
r"[A-Za-z0-9._:-]{1,128}", idempotency_key
|
||||
):
|
||||
raise invalid()
|
||||
return self.repository.add_message(
|
||||
message=Message(
|
||||
uuid4(),
|
||||
chat_id,
|
||||
actor_id,
|
||||
text,
|
||||
utcnow(),
|
||||
tuple(attachment_ids),
|
||||
),
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
|
||||
def chats(self, *, actor_id: UUID, limit: int, cursor: str | None) -> dict:
|
||||
scope = f"chats:{actor_id}"
|
||||
after = decode_cursor(value=cursor, scope=scope)
|
||||
rows = self.repository.list_chats(
|
||||
actor_id=actor_id, after=after, limit=limit
|
||||
)
|
||||
return page(rows=rows, limit=limit, scope=scope)
|
||||
|
||||
def history(
|
||||
self, *, actor_id: UUID, chat: Chat, limit: int, cursor: str | None
|
||||
) -> dict:
|
||||
if actor_id not in chat.member_ids:
|
||||
raise not_found()
|
||||
chat_id = chat.id
|
||||
scope = f"messages:{chat_id}"
|
||||
after = decode_cursor(value=cursor, scope=scope)
|
||||
rows = self.repository.list_messages(
|
||||
actor_id=actor_id, chat_id=chat_id, after=after, limit=limit
|
||||
)
|
||||
return page(rows=rows, limit=limit, scope=scope)
|
||||
@@ -0,0 +1,37 @@
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
file = os.environ.get(name + "_FILE")
|
||||
if file:
|
||||
return Path(file).read_text().strip()
|
||||
value = os.environ.get(name)
|
||||
if not value:
|
||||
raise ValueError("Missing required secret setting: " + name)
|
||||
return value
|
||||
|
||||
|
||||
def integer(name: str, default: int, low: int, high: int) -> int:
|
||||
value = int(os.environ.get(name, str(default)))
|
||||
if not low <= value <= high:
|
||||
raise ValueError("Invalid setting: " + name)
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
database_url: str = field(repr=False)
|
||||
cache_url: str = field(repr=False)
|
||||
session_ttl: int = 3600
|
||||
pool_size: int = 10
|
||||
|
||||
@classmethod
|
||||
def load(cls):
|
||||
return cls(
|
||||
secret("DATABASE_URL"),
|
||||
secret("CACHE_URL"),
|
||||
integer("SESSION_TTL_SECONDS", 3600, 1, 86400),
|
||||
integer("DB_POOL_SIZE", 10, 1, 32),
|
||||
)
|
||||
@@ -0,0 +1,441 @@
|
||||
"""Explicit OTLP signals; no HTTP bodies, credentials, SQL or raw URLs."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import psutil
|
||||
from opentelemetry import trace
|
||||
from opentelemetry._logs import SeverityNumber
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import (
|
||||
OTLPLogExporter,
|
||||
)
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
|
||||
OTLPMetricExporter,
|
||||
)
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
||||
OTLPSpanExporter,
|
||||
)
|
||||
from opentelemetry.metrics import Observation
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import Histogram, MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import (
|
||||
ExplicitBucketHistogramAggregation,
|
||||
View,
|
||||
)
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from opentelemetry.trace.propagation.tracecontext import (
|
||||
TraceContextTextMapPropagator,
|
||||
)
|
||||
|
||||
_state = None
|
||||
_lock = threading.Lock()
|
||||
_pools = weakref.WeakSet()
|
||||
_instruments = {}
|
||||
_observer = {}
|
||||
_propagator = TraceContextTextMapPropagator()
|
||||
BUCKETS = (
|
||||
0.005,
|
||||
0.01,
|
||||
0.025,
|
||||
0.05,
|
||||
0.1,
|
||||
0.25,
|
||||
0.5,
|
||||
1,
|
||||
2.5,
|
||||
5,
|
||||
10,
|
||||
20,
|
||||
60,
|
||||
120,
|
||||
)
|
||||
LOG_FIELDS = {
|
||||
"request_id",
|
||||
"job_id",
|
||||
"attachment_id",
|
||||
"method",
|
||||
"route",
|
||||
"status",
|
||||
"duration_seconds",
|
||||
"attempt",
|
||||
"version",
|
||||
"fence",
|
||||
"outcome",
|
||||
"error_code",
|
||||
"operation",
|
||||
"cycle",
|
||||
}
|
||||
TRACE_FIELDS = {
|
||||
"request_id",
|
||||
"job.id",
|
||||
"attachment.id",
|
||||
"pipeline.version",
|
||||
"job.attempt",
|
||||
"db.system.name",
|
||||
"operation",
|
||||
"messaging.system",
|
||||
"messaging.destination.name",
|
||||
"http.route",
|
||||
"http.request.method",
|
||||
"http.response.status_code",
|
||||
"error.type",
|
||||
}
|
||||
|
||||
|
||||
def enabled():
|
||||
return os.getenv("OTEL_ENABLED", "0") == "1"
|
||||
|
||||
|
||||
def register_pool(pool):
|
||||
_pools.add(pool)
|
||||
|
||||
|
||||
def observe(values):
|
||||
_observer.update(values)
|
||||
|
||||
|
||||
def _pool_stat(key):
|
||||
stats = [p.get_stats() for p in list(_pools)]
|
||||
if key == "used":
|
||||
return sum(
|
||||
max(0, s.get("pool_size", 0) - s.get("pool_available", 0))
|
||||
for s in stats
|
||||
)
|
||||
return sum(s.get("requests_waiting", 0) for s in stats)
|
||||
|
||||
|
||||
def configure(service, *, instance=None, with_metrics=True):
|
||||
global _state
|
||||
if not enabled():
|
||||
return
|
||||
with _lock:
|
||||
if _state is not None:
|
||||
return
|
||||
instance = instance or uuid4().hex
|
||||
resource = Resource.create(
|
||||
{
|
||||
"service.name": service,
|
||||
"service.version": os.getenv("APP_VERSION", "development"),
|
||||
"service.instance.id": instance,
|
||||
"deployment.environment.name": "lab",
|
||||
}
|
||||
)
|
||||
endpoint = os.getenv(
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel:4318"
|
||||
).rstrip("/")
|
||||
options = {"timeout": 0.75}
|
||||
# Certificate options are also honored by the exporters in the TLS lab.
|
||||
mp = None
|
||||
if with_metrics:
|
||||
reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(
|
||||
endpoint=endpoint + "/v1/metrics", **options
|
||||
),
|
||||
export_interval_millis=2000,
|
||||
export_timeout_millis=1000,
|
||||
)
|
||||
mp = MeterProvider(
|
||||
resource=resource,
|
||||
metric_readers=[reader],
|
||||
views=[
|
||||
View(
|
||||
instrument_type=Histogram,
|
||||
aggregation=ExplicitBucketHistogramAggregation(
|
||||
BUCKETS
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
meter = mp.get_meter("messenger", "1.0")
|
||||
for name in ["http_requests", "job_attempts"]:
|
||||
_instruments[name] = meter.create_counter("messenger_" + name)
|
||||
_instruments["worker_busy"] = meter.create_up_down_counter(
|
||||
"messenger_worker_busy"
|
||||
)
|
||||
for name in ["http_duration", "job_duration", "time_to_ready"]:
|
||||
_instruments[name] = meter.create_histogram(
|
||||
"messenger_" + name, unit="s"
|
||||
)
|
||||
for key in ["used", "waiting"]:
|
||||
meter.create_observable_gauge(
|
||||
"messenger_db_pool_" + key,
|
||||
callbacks=[lambda _, k=key: [Observation(_pool_stat(k))]],
|
||||
)
|
||||
process = psutil.Process()
|
||||
meter.create_observable_counter(
|
||||
"messenger_process_cpu",
|
||||
unit="s",
|
||||
callbacks=[
|
||||
lambda _: [Observation(sum(process.cpu_times()[:2]))]
|
||||
],
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
"messenger_process_memory",
|
||||
unit="By",
|
||||
callbacks=[lambda _: [Observation(process.memory_info().rss)]],
|
||||
)
|
||||
|
||||
def container_memory(_):
|
||||
p = Path("/sys/fs/cgroup/memory.current")
|
||||
return [Observation(int(p.read_text()))] if p.exists() else []
|
||||
|
||||
meter.create_observable_gauge(
|
||||
"messenger_container_memory",
|
||||
unit="By",
|
||||
callbacks=[container_memory],
|
||||
)
|
||||
if service == "messenger-worker":
|
||||
meter.create_observable_gauge(
|
||||
"messenger_worker_capacity",
|
||||
callbacks=[
|
||||
lambda _: [
|
||||
Observation(
|
||||
int(os.getenv("WORKER_CONCURRENCY", "2"))
|
||||
)
|
||||
]
|
||||
],
|
||||
)
|
||||
if service == "messenger-observer":
|
||||
for key in ["queue_depth", "oldest_job_age"]:
|
||||
meter.create_observable_gauge(
|
||||
"messenger_" + key,
|
||||
unit="s" if key.endswith("age") else "",
|
||||
callbacks=[
|
||||
lambda _, k=key: [
|
||||
Observation(
|
||||
_observer.get(k, 0),
|
||||
{"queue": "attachments"},
|
||||
)
|
||||
]
|
||||
],
|
||||
)
|
||||
for name, prefix in [
|
||||
("slo_decisions", "slo_"),
|
||||
("jobs_completed", "completed_"),
|
||||
]:
|
||||
meter.create_observable_counter(
|
||||
"messenger_" + name,
|
||||
callbacks=[
|
||||
lambda _, p=prefix: [
|
||||
Observation(
|
||||
_observer.get(p + outcome, 0),
|
||||
{
|
||||
"outcome": outcome,
|
||||
"job_type": "attachment.process.v1",
|
||||
},
|
||||
)
|
||||
for outcome in (
|
||||
["good", "bad"]
|
||||
if p == "slo_"
|
||||
else ["ready", "failed"]
|
||||
)
|
||||
]
|
||||
],
|
||||
)
|
||||
tp = TracerProvider(
|
||||
resource=resource,
|
||||
sampler=ParentBased(
|
||||
TraceIdRatioBased(float(os.getenv("OTEL_TRACE_RATIO", "1")))
|
||||
),
|
||||
)
|
||||
tp.add_span_processor(
|
||||
BatchSpanProcessor(
|
||||
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", **options),
|
||||
max_queue_size=512,
|
||||
max_export_batch_size=64,
|
||||
schedule_delay_millis=200,
|
||||
export_timeout_millis=1000,
|
||||
)
|
||||
)
|
||||
lp = LoggerProvider(resource=resource)
|
||||
lp.add_log_record_processor(
|
||||
BatchLogRecordProcessor(
|
||||
OTLPLogExporter(endpoint=endpoint + "/v1/logs", **options),
|
||||
max_queue_size=512,
|
||||
max_export_batch_size=64,
|
||||
schedule_delay_millis=200,
|
||||
export_timeout_millis=1000,
|
||||
)
|
||||
)
|
||||
_state = SimpleNamespace(
|
||||
service=service,
|
||||
instance=instance,
|
||||
traces=tp,
|
||||
metrics=mp,
|
||||
logs=lp,
|
||||
tracer=tp.get_tracer("messenger", "1.0"),
|
||||
logger=lp.get_logger("messenger", "1.0"),
|
||||
)
|
||||
|
||||
|
||||
def stop():
|
||||
if _state is None:
|
||||
return
|
||||
for provider in [_state.traces, _state.logs, _state.metrics]:
|
||||
if provider is not None:
|
||||
provider.force_flush(timeout_millis=1000)
|
||||
provider.shutdown()
|
||||
|
||||
|
||||
def count(name, value=1, attributes=None):
|
||||
instrument = _instruments.get(name)
|
||||
if instrument is not None:
|
||||
instrument.add(value, attributes or {})
|
||||
|
||||
|
||||
def duration(name, seconds, attributes=None):
|
||||
instrument = _instruments.get(name)
|
||||
if instrument is not None:
|
||||
instrument.record(max(0, seconds), attributes or {})
|
||||
|
||||
|
||||
def carrier():
|
||||
result = {}
|
||||
_propagator.inject(result)
|
||||
if len(result.get("tracestate", "")) > 512:
|
||||
result.pop("tracestate", None)
|
||||
return result
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name, *, kind=SpanKind.INTERNAL, parent=None, attributes=None):
|
||||
tracer = (
|
||||
_state.tracer if _state is not None else trace.get_tracer("messenger")
|
||||
)
|
||||
attrs = {k: v for k, v in (attributes or {}).items() if k in TRACE_FIELDS}
|
||||
context = _propagator.extract(parent) if parent is not None else None
|
||||
with tracer.start_as_current_span(
|
||||
name,
|
||||
kind=kind,
|
||||
context=context,
|
||||
attributes=attrs,
|
||||
record_exception=False,
|
||||
set_status_on_exception=False,
|
||||
) as current:
|
||||
try:
|
||||
yield current
|
||||
except BaseException as exc:
|
||||
current.set_attribute("error.type", type(exc).__name__)
|
||||
current.set_status(Status(StatusCode.ERROR, "operation failed"))
|
||||
raise
|
||||
|
||||
|
||||
def log(event, *, error=False, **fields):
|
||||
event = (
|
||||
event
|
||||
if re.fullmatch(r"[a-z][a-z0-9_.-]{0,79}", event)
|
||||
else "application.event"
|
||||
)
|
||||
attrs = {
|
||||
k: v[:256] if isinstance(v, str) else v
|
||||
for k, v in fields.items()
|
||||
if k in LOG_FIELDS and isinstance(v, (str, int, float, bool))
|
||||
}
|
||||
current = trace.get_current_span().get_span_context()
|
||||
record = {
|
||||
"event": event,
|
||||
"timestamp": time.time(),
|
||||
"severity": "ERROR" if error else "INFO",
|
||||
"service.name": _state.service if _state else "messenger",
|
||||
**attrs,
|
||||
}
|
||||
if current.is_valid:
|
||||
record.update(
|
||||
trace_id=f"{current.trace_id:032x}",
|
||||
span_id=f"{current.span_id:016x}",
|
||||
)
|
||||
print(json.dumps(record, ensure_ascii=False), flush=True)
|
||||
if _state is not None:
|
||||
_state.logger.emit(
|
||||
body=event,
|
||||
attributes=attrs,
|
||||
severity_text=record["severity"],
|
||||
severity_number=SeverityNumber.ERROR
|
||||
if error
|
||||
else SeverityNumber.INFO,
|
||||
)
|
||||
|
||||
|
||||
class TelemetryMiddleware:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
# Health polling does not enter business SLIs or spans.
|
||||
if scope.get("path", "").startswith("/health/"):
|
||||
return await self.app(scope, receive, send)
|
||||
headers = {
|
||||
k.decode("latin1"): v.decode("latin1")
|
||||
for k, v in scope.get("headers", [])
|
||||
if k.lower() in {b"traceparent", b"tracestate"}
|
||||
}
|
||||
status = 500
|
||||
started = time.monotonic()
|
||||
|
||||
async def measured_send(message):
|
||||
nonlocal status
|
||||
if message["type"] == "http.response.start":
|
||||
status = message["status"]
|
||||
await send(message)
|
||||
|
||||
with span("HTTP", kind=SpanKind.SERVER, parent=headers) as current:
|
||||
try:
|
||||
await self.app(scope, receive, measured_send)
|
||||
finally:
|
||||
route = getattr(scope.get("route"), "path", "unmatched")
|
||||
method = scope.get("method", "UNKNOWN")
|
||||
if method not in {
|
||||
"GET",
|
||||
"POST",
|
||||
"PUT",
|
||||
"PATCH",
|
||||
"DELETE",
|
||||
"HEAD",
|
||||
"OPTIONS",
|
||||
}:
|
||||
method = "OTHER"
|
||||
current.update_name(method + " " + route)
|
||||
current.set_attributes(
|
||||
{
|
||||
"http.route": route,
|
||||
"http.request.method": method,
|
||||
"http.response.status_code": status,
|
||||
}
|
||||
)
|
||||
if status >= 500:
|
||||
current.set_status(
|
||||
Status(StatusCode.ERROR, "HTTP server error")
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
labels = {
|
||||
"route": route,
|
||||
"method": method,
|
||||
"status_class": str(status // 100) + "xx",
|
||||
}
|
||||
count("http_requests", attributes=labels)
|
||||
duration("http_duration", elapsed, labels)
|
||||
log(
|
||||
"http.request",
|
||||
request_id=scope.get("state", {}).get("request_id", ""),
|
||||
method=method,
|
||||
route=route,
|
||||
status=status,
|
||||
duration_seconds=round(elapsed, 6),
|
||||
error=status >= 500,
|
||||
)
|
||||
+547
@@ -0,0 +1,547 @@
|
||||
"""Two bounded consumers; every decode runs in a killable isolated process."""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import multiprocessing
|
||||
import resource
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import redis
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
from src import telemetry
|
||||
from src.images import MAX_BYTES, MAX_PIXELS
|
||||
from src.object_store import ObjectStore
|
||||
from src.pipeline import (
|
||||
GROUP,
|
||||
STREAM,
|
||||
PermanentFailure,
|
||||
broker,
|
||||
database,
|
||||
event,
|
||||
fault,
|
||||
retry,
|
||||
validate_job,
|
||||
)
|
||||
from src.settings import integer
|
||||
|
||||
|
||||
def sizes(width, height, bound):
|
||||
longest = max(width, height)
|
||||
return (
|
||||
(width, height)
|
||||
if longest <= bound
|
||||
else (
|
||||
max(1, width * bound // longest),
|
||||
max(1, height * bound // longest),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def make_variants(raw, row):
|
||||
if (
|
||||
len(raw) != row["size_bytes"]
|
||||
or hashlib.sha256(raw).hexdigest() != row["sha256"]
|
||||
):
|
||||
raise PermanentFailure("source_checksum_mismatch")
|
||||
try:
|
||||
with Image.open(io.BytesIO(raw)) as source:
|
||||
if source.width * source.height > MAX_PIXELS:
|
||||
raise PermanentFailure("pixel_limit")
|
||||
if source.format not in {"PNG", "JPEG"}:
|
||||
raise PermanentFailure("invalid_image")
|
||||
source.load()
|
||||
oriented = ImageOps.exif_transpose(source)
|
||||
rgba = oriented.convert("RGBA")
|
||||
rgb = Image.new("RGB", rgba.size, "white")
|
||||
rgb.paste(rgba, mask=rgba.getchannel("A"))
|
||||
rgb.info.clear()
|
||||
yield "original", raw, row["content_type"], source.size
|
||||
thumbnail = rgb.resize(
|
||||
sizes(*rgb.size, 256), Image.Resampling.LANCZOS
|
||||
)
|
||||
crop = ImageOps.fit(
|
||||
rgb, (128, 128), Image.Resampling.LANCZOS, centering=(0.5, 0.5)
|
||||
)
|
||||
optimized = rgb.resize(
|
||||
sizes(*rgb.size, 1280), Image.Resampling.LANCZOS
|
||||
)
|
||||
for name, image, fmt, mime, options in [
|
||||
(
|
||||
"thumbnail",
|
||||
thumbnail,
|
||||
"JPEG",
|
||||
"image/jpeg",
|
||||
{"quality": 85, "subsampling": 0},
|
||||
),
|
||||
(
|
||||
"crop",
|
||||
crop,
|
||||
"JPEG",
|
||||
"image/jpeg",
|
||||
{"quality": 85, "subsampling": 0},
|
||||
),
|
||||
(
|
||||
"optimized",
|
||||
optimized,
|
||||
"WEBP",
|
||||
"image/webp",
|
||||
{
|
||||
"quality": 82 if row["pipeline_version"] == 1 else 76,
|
||||
"method": 4,
|
||||
},
|
||||
),
|
||||
]:
|
||||
output = io.BytesIO()
|
||||
image.info.clear()
|
||||
image.save(output, format=fmt, **options)
|
||||
yield name, output.getvalue(), mime, image.size
|
||||
except (
|
||||
UnidentifiedImageError,
|
||||
OSError,
|
||||
ValueError,
|
||||
Image.DecompressionBombError,
|
||||
) as exc:
|
||||
raise PermanentFailure("invalid_image") from exc
|
||||
|
||||
|
||||
def process(job_id, fence, sender, trace_context=None):
|
||||
telemetry.configure("messenger-worker", with_metrics=False)
|
||||
try:
|
||||
with telemetry.span(
|
||||
"attachment.process",
|
||||
parent=trace_context,
|
||||
attributes={"job.id": job_id},
|
||||
):
|
||||
telemetry.log("job.processing", job_id=job_id, fence=fence)
|
||||
process_body(job_id, fence, sender)
|
||||
finally:
|
||||
telemetry.stop()
|
||||
|
||||
|
||||
def process_body(job_id, fence, sender):
|
||||
db = store = None
|
||||
started = time.monotonic()
|
||||
cpu = time.process_time()
|
||||
try:
|
||||
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
|
||||
db, store = database(), ObjectStore()
|
||||
with db.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT j.*,a.size_bytes,a.sha256,a.content_type FROM jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s AND j.fence=%s",
|
||||
(job_id, fence),
|
||||
).fetchone()
|
||||
if not row:
|
||||
sender.send({"result": "stale"})
|
||||
return
|
||||
# Record limits before fault hooks: a slow external inspector must not
|
||||
# consume the child timeout while attempting to observe the limit.
|
||||
cgroup = Path("/sys/fs/cgroup")
|
||||
limits = {
|
||||
name: (cgroup / name).read_text().strip()
|
||||
if (cgroup / name).exists()
|
||||
else "unavailable"
|
||||
for name in ["memory.max", "cpu.max"]
|
||||
}
|
||||
limits["address_space_bytes"] = list(
|
||||
resource.getrlimit(resource.RLIMIT_AS)
|
||||
)
|
||||
with db.connection() as conn:
|
||||
event(conn, job_id, "worker_resource_limits", limits)
|
||||
fault(db, job_id, "before_effect")
|
||||
upstream = store.get(key=row["source_key"])
|
||||
try:
|
||||
raw = upstream.read(MAX_BYTES + 1)
|
||||
finally:
|
||||
upstream.close()
|
||||
upstream.release_conn()
|
||||
results = []
|
||||
for name, data, mime, (width, height) in make_variants(raw, row):
|
||||
key = f"objects/{row['attachment_id']}/v{row['pipeline_version']}/{name}"
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
store.put(
|
||||
key=key,
|
||||
stream=io.BytesIO(data),
|
||||
size=len(data),
|
||||
content_type=mime,
|
||||
sha256=digest,
|
||||
)
|
||||
results.append((name, key, mime, len(data), digest, width, height))
|
||||
fault(db, job_id, "after_effect")
|
||||
with db.connection() as conn, conn.transaction():
|
||||
current = conn.execute(
|
||||
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
|
||||
).fetchone()
|
||||
allowed = conn.execute(
|
||||
"SELECT pipeline_version FROM attachments WHERE id=%s FOR UPDATE",
|
||||
(row["attachment_id"],),
|
||||
).fetchone()
|
||||
valid_lease = conn.execute(
|
||||
"SELECT %s::timestamptz>clock_timestamp() live",
|
||||
(current["lease_until"],),
|
||||
).fetchone()["live"]
|
||||
if (
|
||||
current["state"] != "processing"
|
||||
or current["fence"] != fence
|
||||
or not valid_lease
|
||||
or allowed["pipeline_version"] != row["pipeline_version"]
|
||||
):
|
||||
event(
|
||||
conn,
|
||||
job_id,
|
||||
"stale_completion_rejected",
|
||||
{"fence": fence, "version": row["pipeline_version"]},
|
||||
)
|
||||
sender.send({"result": "stale"})
|
||||
return
|
||||
for result in results:
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT(attachment_id,pipeline_version,name) DO NOTHING",
|
||||
(row["attachment_id"], row["pipeline_version"], *result),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO effect_publications(attachment_id,pipeline_version,job_id,fence) VALUES(%s,%s,%s,%s)",
|
||||
(row["attachment_id"], row["pipeline_version"], job_id, fence),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state='ready',object_key=%s,published_version=pipeline_version,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(results[0][1], row["attachment_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state='ready',error_code=NULL,lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(job_id,),
|
||||
)
|
||||
event(
|
||||
conn,
|
||||
job_id,
|
||||
"effect_committed",
|
||||
{
|
||||
"fence": fence,
|
||||
"version": row["pipeline_version"],
|
||||
"variants": 4,
|
||||
},
|
||||
)
|
||||
fault(db, job_id, "after_commit")
|
||||
# Cleanup is independently retryable; it cannot undo the committed effect.
|
||||
if row["source_key"].startswith("objects/staging/"):
|
||||
try:
|
||||
store.delete(key=row["source_key"])
|
||||
except Exception:
|
||||
pass
|
||||
telemetry.log(
|
||||
"job.ready", job_id=job_id, version=row["pipeline_version"]
|
||||
)
|
||||
sender.send(
|
||||
{
|
||||
"result": "ready",
|
||||
"seconds": time.monotonic() - started,
|
||||
"cpu_seconds": time.process_time() - cpu,
|
||||
"rss_peak_mib": resource.getrusage(
|
||||
resource.RUSAGE_SELF
|
||||
).ru_maxrss
|
||||
/ 1024,
|
||||
}
|
||||
)
|
||||
except PermanentFailure as exc:
|
||||
telemetry.log(
|
||||
"job.processing_failed",
|
||||
job_id=job_id,
|
||||
error=True,
|
||||
error_code=str(exc),
|
||||
)
|
||||
telemetry.trace.get_current_span().set_status(
|
||||
telemetry.Status(
|
||||
telemetry.StatusCode.ERROR, "permanent processing error"
|
||||
)
|
||||
)
|
||||
sender.send({"result": "permanent", "code": str(exc)})
|
||||
except MemoryError:
|
||||
telemetry.log(
|
||||
"job.processing_failed",
|
||||
job_id=job_id,
|
||||
error=True,
|
||||
error_code="memory_limit",
|
||||
)
|
||||
telemetry.trace.get_current_span().set_status(
|
||||
telemetry.Status(telemetry.StatusCode.ERROR, "memory limit")
|
||||
)
|
||||
sender.send({"result": "permanent", "code": "memory_limit"})
|
||||
except Exception:
|
||||
telemetry.log(
|
||||
"job.processing_failed",
|
||||
job_id=job_id,
|
||||
error=True,
|
||||
error_code="processing_unavailable",
|
||||
)
|
||||
telemetry.trace.get_current_span().set_status(
|
||||
telemetry.Status(
|
||||
telemetry.StatusCode.ERROR, "transient processing error"
|
||||
)
|
||||
)
|
||||
sender.send({"result": "transient", "code": "processing_unavailable"})
|
||||
finally:
|
||||
if store:
|
||||
store.close()
|
||||
if db:
|
||||
db.close()
|
||||
sender.close()
|
||||
|
||||
|
||||
def claim(db, data, consumer):
|
||||
with db.connection() as conn, conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (data["job_id"],)
|
||||
).fetchone()
|
||||
if (
|
||||
not row
|
||||
or str(row["attachment_id"]) != data["attachment_id"]
|
||||
or row["pipeline_version"] != data["pipeline_version"]
|
||||
):
|
||||
return "obsolete", None
|
||||
if row["state"] in {"ready", "failed", "obsolete"}:
|
||||
return row["state"], row
|
||||
if data["attempt"] < row["attempt"] or (
|
||||
row["state"] == "queued" and data["attempt"] <= row["attempt"]
|
||||
):
|
||||
return "obsolete", row
|
||||
if row["state"] == "processing":
|
||||
return "busy", row
|
||||
if data["attempt"] != row["attempt"] + 1:
|
||||
return "obsolete", row
|
||||
due = conn.execute(
|
||||
"SELECT %s::timestamptz<=clock_timestamp() due", (row["next_at"],)
|
||||
).fetchone()["due"]
|
||||
if not due:
|
||||
return "busy", row
|
||||
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
|
||||
updated = conn.execute(
|
||||
"UPDATE jobs SET state='processing',attempt=attempt+1,fence=fence+1,lease_owner=%s,lease_until=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s RETURNING *",
|
||||
(consumer, lease, row["id"]),
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
|
||||
(row["attachment_id"], row["pipeline_version"]),
|
||||
)
|
||||
event(
|
||||
conn,
|
||||
row["id"],
|
||||
"claimed",
|
||||
{
|
||||
"attempt": updated["attempt"],
|
||||
"fence": updated["fence"],
|
||||
"version": updated["pipeline_version"],
|
||||
},
|
||||
)
|
||||
return "claimed", updated
|
||||
|
||||
|
||||
def ready_effect(db, store, row):
|
||||
with db.connection() as conn:
|
||||
variants = conn.execute(
|
||||
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s",
|
||||
(row["attachment_id"], row["pipeline_version"]),
|
||||
).fetchall()
|
||||
if len(variants) != 4:
|
||||
return False
|
||||
for variant in variants:
|
||||
head = store.head(key=variant["object_key"])
|
||||
if (
|
||||
head is None
|
||||
or head.size != variant["size_bytes"]
|
||||
or head.metadata.get("x-amz-meta-sha256") != variant["sha256"]
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def consume(slot):
|
||||
db, queue, store = database(), broker(), ObjectStore()
|
||||
consumer = str(uuid4()) + f"-{slot}"
|
||||
context = multiprocessing.get_context("spawn")
|
||||
cursor = "0-0"
|
||||
timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117)
|
||||
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
|
||||
if lease <= timeout:
|
||||
raise ValueError("Lease must exceed child timeout")
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
try:
|
||||
queue.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if "BUSYGROUP" not in str(exc):
|
||||
raise
|
||||
reclaimed = queue.xautoclaim(
|
||||
STREAM,
|
||||
GROUP,
|
||||
consumer,
|
||||
# Do not steal an entry from a consumer inside its live budget.
|
||||
min_idle_time=lease * 1000,
|
||||
start_id=cursor,
|
||||
count=1,
|
||||
)
|
||||
cursor, messages = reclaimed[0], reclaimed[1]
|
||||
if not messages:
|
||||
batch = queue.xreadgroup(
|
||||
GROUP, consumer, {STREAM: ">"}, count=1, block=500
|
||||
)
|
||||
messages = batch[0][1] if batch else []
|
||||
for entry, fields in messages:
|
||||
try:
|
||||
data = validate_job(
|
||||
json.loads(fields.get("job", "null"))
|
||||
)
|
||||
except ValueError, TypeError, KeyError:
|
||||
# Malformed notifications cannot become jobs; preserve diagnostic count.
|
||||
with db.connection() as conn:
|
||||
event(conn, None, "invalid_envelope")
|
||||
queue.xack(STREAM, GROUP, entry)
|
||||
queue.xdel(STREAM, entry)
|
||||
continue
|
||||
with telemetry.span(
|
||||
"attachments consume",
|
||||
kind=telemetry.SpanKind.CONSUMER,
|
||||
parent=data,
|
||||
attributes={
|
||||
"job.id": data["job_id"],
|
||||
"messaging.system": "valkey",
|
||||
"messaging.destination.name": STREAM,
|
||||
},
|
||||
):
|
||||
state, row = claim(db, data, consumer)
|
||||
if state == "busy":
|
||||
continue
|
||||
if state == "ready" and not ready_effect(
|
||||
db, store, row
|
||||
):
|
||||
with db.connection() as conn:
|
||||
event(conn, row["id"], "ready_effect_missing")
|
||||
continue
|
||||
if state in {"ready", "obsolete", "failed"}:
|
||||
with db.connection() as conn:
|
||||
event(
|
||||
conn,
|
||||
data["job_id"],
|
||||
"duplicate_decision",
|
||||
{"state": state, "entry": entry},
|
||||
)
|
||||
if state == "claimed":
|
||||
attempt_started = time.monotonic()
|
||||
receiver, sender = context.Pipe(duplex=False)
|
||||
child = context.Process(
|
||||
target=process,
|
||||
args=(
|
||||
str(row["id"]),
|
||||
row["fence"],
|
||||
sender,
|
||||
telemetry.carrier(),
|
||||
),
|
||||
)
|
||||
child.start()
|
||||
telemetry.count("worker_busy", 1)
|
||||
sender.close()
|
||||
child.join(timeout)
|
||||
if child.is_alive():
|
||||
child.kill()
|
||||
child.join(2)
|
||||
result = {
|
||||
"result": "transient",
|
||||
"code": "worker_timeout",
|
||||
}
|
||||
else:
|
||||
try:
|
||||
result = (
|
||||
receiver.recv()
|
||||
if receiver.poll()
|
||||
else {
|
||||
"result": "transient",
|
||||
"code": "worker_crashed",
|
||||
}
|
||||
)
|
||||
except EOFError:
|
||||
result = {
|
||||
"result": "transient",
|
||||
"code": "worker_crashed",
|
||||
}
|
||||
telemetry.count("worker_busy", -1)
|
||||
labels = {
|
||||
"job_type": "attachment.process.v1",
|
||||
"outcome": result["result"],
|
||||
}
|
||||
telemetry.count("job_attempts", attributes=labels)
|
||||
telemetry.duration(
|
||||
"job_duration",
|
||||
time.monotonic() - attempt_started,
|
||||
labels,
|
||||
)
|
||||
telemetry.log(
|
||||
"job.attempt_finished",
|
||||
job_id=str(row["id"]),
|
||||
attempt=row["attempt"],
|
||||
version=row["pipeline_version"],
|
||||
outcome=result["result"],
|
||||
error_code=result.get("code", ""),
|
||||
)
|
||||
receiver.close()
|
||||
child.close()
|
||||
with db.connection() as conn, conn.transaction():
|
||||
current = conn.execute(
|
||||
"SELECT * FROM jobs WHERE id=%s FOR UPDATE",
|
||||
(row["id"],),
|
||||
).fetchone()
|
||||
event(
|
||||
conn, row["id"], "child_finished", result
|
||||
)
|
||||
if (
|
||||
current["state"] == "processing"
|
||||
and current["fence"] == row["fence"]
|
||||
):
|
||||
retry(
|
||||
conn,
|
||||
current,
|
||||
result.get(
|
||||
"code", "processing_failed"
|
||||
),
|
||||
permanent=result["result"]
|
||||
== "permanent",
|
||||
)
|
||||
# Commit result or a retry/terminal decision precedes ack.
|
||||
queue.xack(STREAM, GROUP, entry)
|
||||
queue.xdel(STREAM, entry)
|
||||
except Exception:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "worker_retry",
|
||||
"code": "dependency_unavailable",
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
queue.close()
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
def main():
|
||||
telemetry.configure("messenger-worker")
|
||||
count = integer("WORKER_CONCURRENCY", 2, 1, 4)
|
||||
threads = [
|
||||
threading.Thread(target=consume, args=(i,), daemon=False)
|
||||
for i in range(count)
|
||||
]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
telemetry.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,119 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator, FormatChecker
|
||||
|
||||
CONTRACT = json.loads(
|
||||
(Path(__file__).parents[1] / "contracts/openapi.json").read_text()
|
||||
)
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8086")
|
||||
PASSWORD = "Synthetic-Password-🌍-123"
|
||||
TOKENS = {}
|
||||
|
||||
|
||||
def check(result: httpx.Response, status: int, schema: str) -> dict:
|
||||
assert result.status_code == status, result.text
|
||||
assert result.headers["x-request-id"]
|
||||
assert result.headers["x-instance-id"]
|
||||
assert result.headers["content-type"].startswith("application/json")
|
||||
payload = result.json()
|
||||
spec = {"$ref": f"#/components/schemas/{schema}", **CONTRACT}
|
||||
Draft202012Validator(spec, format_checker=FormatChecker()).validate(
|
||||
payload
|
||||
)
|
||||
if schema == "Error":
|
||||
assert payload["error"]["request_id"] == result.headers["x-request-id"]
|
||||
assert (
|
||||
payload["error"]["code"]
|
||||
== {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
500: "internal_error",
|
||||
503: "unavailable",
|
||||
}[status]
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def register(client: httpx.Client, **overrides) -> dict:
|
||||
data = {
|
||||
"username": "u_" + uuid4().hex[:24],
|
||||
"display_name": "Алиса 🌍",
|
||||
"password": PASSWORD,
|
||||
}
|
||||
data.update(overrides)
|
||||
user = check(client.post("/api/v1/users", json=data), 201, "User")
|
||||
session = check(
|
||||
client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
auth=httpx.BasicAuth(data["username"], data["password"]),
|
||||
),
|
||||
201,
|
||||
"Session",
|
||||
)
|
||||
TOKENS[user["id"]] = session["token"]
|
||||
return user
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
with httpx.Client(
|
||||
base_url=BASE_URL, timeout=15, trust_env=False
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def actors(client):
|
||||
return register(client), register(client), register(client)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def chat(client, actors):
|
||||
alice, bob, _ = actors
|
||||
return check(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(alice),
|
||||
json={"title": "Учёба 🌍", "member_ids": [alice["id"], bob["id"]]},
|
||||
),
|
||||
201,
|
||||
"Chat",
|
||||
)
|
||||
|
||||
|
||||
def headers(actor: dict) -> dict:
|
||||
return {
|
||||
"Authorization": "Bearer " + TOKENS[actor["id"]],
|
||||
"Idempotency-Key": uuid4().hex,
|
||||
}
|
||||
|
||||
|
||||
def history(client, actor, chat_id, limit=7):
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
params = {"limit": limit}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
page = check(
|
||||
client.get(
|
||||
f"/api/v1/chats/{chat_id}/messages",
|
||||
headers=headers(actor),
|
||||
params=params,
|
||||
),
|
||||
200,
|
||||
"MessagePage",
|
||||
)
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return found
|
||||
@@ -0,0 +1,412 @@
|
||||
import base64
|
||||
import json
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import BASE_URL, PASSWORD, check, headers, history, register
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/health/live", "/health/ready"])
|
||||
def test_health(client, path):
|
||||
assert check(client.get(path), 200, "Health") == {"status": "ok"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value", [None, "not-a-uuid", str(uuid4()), uuid4().hex]
|
||||
)
|
||||
def test_identification(client, value):
|
||||
check(
|
||||
client.get(
|
||||
"/api/v1/users/me", headers={"X-User-Id": value} if value else {}
|
||||
),
|
||||
401,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_identity(client, actors):
|
||||
assert (
|
||||
check(
|
||||
client.get("/api/v1/users/me", headers=headers(actors[0])),
|
||||
200,
|
||||
"User",
|
||||
)
|
||||
== actors[0]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data",
|
||||
[
|
||||
{},
|
||||
[],
|
||||
None,
|
||||
{"username": "ABC", "display_name": "A"},
|
||||
{"username": "abc\n", "display_name": "A"},
|
||||
{"username": 123, "display_name": "A"},
|
||||
{"username": "abc", "display_name": " "},
|
||||
{"username": "abc", "display_name": "x" * 101},
|
||||
{"username": "abc", "display_name": "A", "extra": True},
|
||||
{"username": "abc", "display_name": "\ud800"},
|
||||
],
|
||||
)
|
||||
def test_invalid_user(client, data):
|
||||
if isinstance(data, dict):
|
||||
data = {"password": PASSWORD, **data}
|
||||
check(client.post("/api/v1/users", content=json.dumps(data)), 400, "Error")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw", [b"{", b"NaN", b'{"username":"abc", "display_name":NaN}', b"\xff"]
|
||||
)
|
||||
def test_malformed_json(client, raw):
|
||||
check(client.post("/api/v1/users", content=raw), 400, "Error")
|
||||
|
||||
|
||||
def test_auth_before_json(client):
|
||||
check(client.post("/api/v1/chats", content=b"{"), 401, "Error")
|
||||
|
||||
|
||||
def test_duplicate_username(client, actors):
|
||||
check(
|
||||
client.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"username": actors[0]["username"],
|
||||
"display_name": "B",
|
||||
"password": PASSWORD,
|
||||
},
|
||||
),
|
||||
409,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_unicode_codepoints(client, actors, chat):
|
||||
user = register(client, display_name="🌍" * 100)
|
||||
assert user["display_name"] == "🌍" * 100
|
||||
text = "🌍" * 4000
|
||||
result = check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": text},
|
||||
),
|
||||
201,
|
||||
"Message",
|
||||
)
|
||||
assert result["text"] == text
|
||||
assert result["sender_id"] == actors[0]["id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method,suffix,body",
|
||||
[
|
||||
("GET", "", None),
|
||||
("GET", "/messages", None),
|
||||
("POST", "/messages", {"text": "hello"}),
|
||||
],
|
||||
)
|
||||
def test_acl(client, actors, chat, method, suffix, body):
|
||||
outsider = headers(actors[2])
|
||||
actual = check(
|
||||
client.request(
|
||||
method,
|
||||
f"/api/v1/chats/{chat['id']}{suffix}",
|
||||
headers=outsider,
|
||||
json=body,
|
||||
),
|
||||
404,
|
||||
"Error",
|
||||
)
|
||||
missing = check(
|
||||
client.request(
|
||||
method,
|
||||
f"/api/v1/chats/{uuid4()}{suffix}",
|
||||
headers=outsider,
|
||||
json=body,
|
||||
),
|
||||
404,
|
||||
"Error",
|
||||
)
|
||||
assert actual["error"]["message"] == missing["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", " \t\n", 42, True, None, "a" * 4001])
|
||||
def test_bad_message(client, actors, chat, value):
|
||||
check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": value},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_sender_not_client_controlled(client, actors, chat):
|
||||
check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": "Hi", "sender_id": actors[1]["id"]},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
"duplicate",
|
||||
"unknown",
|
||||
"no_caller",
|
||||
"empty",
|
||||
"wrong_type",
|
||||
"bad_uuid",
|
||||
"too_many",
|
||||
"blank_title",
|
||||
],
|
||||
)
|
||||
def test_invalid_chat(client, actors, case):
|
||||
a, b, _ = actors
|
||||
members = {
|
||||
"duplicate": [a["id"], a["id"]],
|
||||
"unknown": [a["id"], str(uuid4())],
|
||||
"no_caller": [b["id"]],
|
||||
"empty": [],
|
||||
"wrong_type": a["id"],
|
||||
"bad_uuid": ["broken"],
|
||||
"too_many": [str(uuid4()) for _ in range(101)],
|
||||
"blank_title": [a["id"]],
|
||||
}[case]
|
||||
check(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(a),
|
||||
json={
|
||||
"title": " " if case == "blank_title" else "Title",
|
||||
"member_ids": members,
|
||||
},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"limit", ["0", "101", "-1", "1.5", "true", "1e1", "", "12"]
|
||||
)
|
||||
def test_bad_limit(client, actors, chat, limit):
|
||||
check(
|
||||
client.get(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
params={"limit": limit},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cursor", ["", "broken", "W10", "e30", "*", "x" * 2049]
|
||||
)
|
||||
def test_bad_cursor(client, actors, chat, cursor):
|
||||
check(
|
||||
client.get(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
params={"cursor": cursor},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_pagination(client, actors, chat):
|
||||
a, b, outsider = actors
|
||||
sent = []
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
for i in range(23):
|
||||
sent.append(
|
||||
check(
|
||||
client.post(
|
||||
path,
|
||||
headers=headers(a if i % 2 else b),
|
||||
json={"text": f"Msg {i}"},
|
||||
),
|
||||
201,
|
||||
"Message",
|
||||
)
|
||||
)
|
||||
got = history(client, a, chat["id"])
|
||||
assert got == sorted(
|
||||
sent, key=lambda m: (m["created_at"], UUID(m["id"]).int)
|
||||
)
|
||||
first = check(
|
||||
client.get(path, headers=headers(a), params={"limit": 1}),
|
||||
200,
|
||||
"MessagePage",
|
||||
)
|
||||
second = check(
|
||||
client.get(
|
||||
path,
|
||||
headers=headers(b),
|
||||
params={"limit": 22, "cursor": first["next_cursor"]},
|
||||
),
|
||||
200,
|
||||
"MessagePage",
|
||||
)
|
||||
assert second["items"] == got[1:]
|
||||
assert second["next_cursor"] is None
|
||||
chats = check(
|
||||
client.get("/api/v1/chats", headers=headers(outsider)), 200, "ChatPage"
|
||||
)
|
||||
assert chats["items"] == []
|
||||
other = check(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(a),
|
||||
json={"title": "Other", "member_ids": [a["id"]]},
|
||||
),
|
||||
201,
|
||||
"Chat",
|
||||
)
|
||||
check(
|
||||
client.get(
|
||||
f"/api/v1/chats/{other['id']}/messages",
|
||||
headers=headers(a),
|
||||
params={"cursor": first["next_cursor"]},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
# Валидный base64/JSON с неверным типом ключа тоже даёт 400.
|
||||
data = json.loads(
|
||||
base64.urlsafe_b64decode(
|
||||
first["next_cursor"] + "=" * (-len(first["next_cursor"]) % 4)
|
||||
)
|
||||
)
|
||||
data["at"] = 17
|
||||
forged = (
|
||||
base64.urlsafe_b64encode(json.dumps(data).encode())
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
)
|
||||
check(
|
||||
client.get(path, headers=headers(a), params={"cursor": forged}),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_chat_pagination_scope(client, actors):
|
||||
a, b, _ = actors
|
||||
created = []
|
||||
for i in range(4):
|
||||
created.append(
|
||||
check(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(a),
|
||||
json={"title": str(i), "member_ids": [a["id"], b["id"]]},
|
||||
),
|
||||
201,
|
||||
"Chat",
|
||||
)
|
||||
)
|
||||
page = check(
|
||||
client.get("/api/v1/chats", headers=headers(a), params={"limit": 2}),
|
||||
200,
|
||||
"ChatPage",
|
||||
)
|
||||
assert page["items"] == created[:2]
|
||||
last = check(
|
||||
client.get(
|
||||
"/api/v1/chats",
|
||||
headers=headers(a),
|
||||
params={"limit": 2, "cursor": page["next_cursor"]},
|
||||
),
|
||||
200,
|
||||
"ChatPage",
|
||||
)
|
||||
assert last["items"] == created[2:]
|
||||
assert last["next_cursor"] is None
|
||||
check(
|
||||
client.get(
|
||||
"/api/v1/chats",
|
||||
headers=headers(b),
|
||||
params={"cursor": page["next_cursor"]},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_messages(client, actors, chat):
|
||||
barrier = threading.Barrier(10)
|
||||
a = actors[0]
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
def worker(i):
|
||||
with httpx.Client(
|
||||
base_url=BASE_URL, timeout=15, trust_env=False
|
||||
) as conn:
|
||||
barrier.wait(timeout=15)
|
||||
return [
|
||||
check(
|
||||
conn.post(
|
||||
path, headers=headers(a), json={"text": f"{i}:{j}"}
|
||||
),
|
||||
201,
|
||||
"Message",
|
||||
)["id"]
|
||||
for j in range(10)
|
||||
]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
ids = [item for batch in pool.map(worker, range(10)) for item in batch]
|
||||
rows = history(client, a, chat["id"])
|
||||
assert len(rows) == len(ids) == len(set(ids)) == 100
|
||||
assert set(ids) == {row["id"] for row in rows}
|
||||
assert len({row["text"] for row in rows}) == 100
|
||||
|
||||
|
||||
def test_registration_race(client):
|
||||
username = "race_" + uuid4().hex[:20]
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def worker(_):
|
||||
with httpx.Client(
|
||||
base_url=BASE_URL, timeout=15, trust_env=False
|
||||
) as conn:
|
||||
barrier.wait(timeout=15)
|
||||
result = conn.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"username": username,
|
||||
"display_name": "Test",
|
||||
"password": PASSWORD,
|
||||
},
|
||||
)
|
||||
check(
|
||||
result,
|
||||
result.status_code,
|
||||
"User" if result.status_code == 201 else "Error",
|
||||
)
|
||||
return result.status_code
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
statuses = list(pool.map(worker, range(10)))
|
||||
assert statuses.count(201) == 1
|
||||
assert statuses.count(409) == 9
|
||||
|
||||
|
||||
def test_unknown_path(client):
|
||||
check(client.get("/does-not-exist"), 404, "Error")
|
||||
@@ -0,0 +1,277 @@
|
||||
import hashlib
|
||||
import io
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from conftest import check, headers
|
||||
from PIL import Image
|
||||
|
||||
PNG = (Path(__file__).parents[1] / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def upload(
|
||||
client, actor, chat, *, data=PNG, filename="Фото 🌍.png", mime="image/png"
|
||||
):
|
||||
return client.post(
|
||||
f"/api/v1/chats/{chat['id']}/attachments",
|
||||
headers={**headers(actor), "Content-Type": mime},
|
||||
params={"filename": filename},
|
||||
content=data,
|
||||
)
|
||||
|
||||
|
||||
def wait_ready(client, actor, attachment):
|
||||
end = time.monotonic() + 60
|
||||
while time.monotonic() < end:
|
||||
row = check(
|
||||
client.get(
|
||||
f"/api/v1/attachments/{attachment['id']}",
|
||||
headers=headers(actor),
|
||||
),
|
||||
200,
|
||||
"Attachment",
|
||||
)
|
||||
if row["status"] == "ready":
|
||||
return row
|
||||
assert row["status"] != "failed", row
|
||||
time.sleep(0.2)
|
||||
raise TimeoutError("Pipeline readiness")
|
||||
|
||||
|
||||
def test_attachment_roundtrip_acl_and_message(client, actors, chat):
|
||||
attachment = check(upload(client, actors[0], chat), 202, "Attachment")
|
||||
assert attachment["status"] == "queued" and attachment["variants"] == []
|
||||
attachment = wait_ready(client, actors[0], attachment)
|
||||
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
|
||||
assert attachment["size_bytes"] == len(PNG)
|
||||
path = f"/api/v1/attachments/{attachment['id']}"
|
||||
assert (
|
||||
check(client.get(path, headers=headers(actors[1])), 200, "Attachment")
|
||||
== attachment
|
||||
)
|
||||
downloaded = client.get(path + "/content", headers=headers(actors[1]))
|
||||
assert downloaded.status_code == 200 and downloaded.content == PNG
|
||||
assert downloaded.headers["Content-Type"] == "image/png"
|
||||
assert not downloaded.history
|
||||
for suffix in ["", "/content"]:
|
||||
check(
|
||||
client.get(path + suffix, headers=headers(actors[2])), 404, "Error"
|
||||
)
|
||||
check(client.get(path + suffix), 401, "Error")
|
||||
thumb = client.get(
|
||||
path + "/content",
|
||||
headers=headers(actors[1]),
|
||||
params={"variant": "thumbnail"},
|
||||
)
|
||||
assert thumb.status_code == 200
|
||||
with Image.open(io.BytesIO(thumb.content)) as image:
|
||||
assert image.format == "JPEG" and image.size == (256, 160)
|
||||
sent = check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[1]),
|
||||
json={"text": "", "attachment_ids": [attachment["id"]]},
|
||||
),
|
||||
201,
|
||||
"Message",
|
||||
)
|
||||
assert sent["attachment_ids"] == [attachment["id"]]
|
||||
assert sent["sender_id"] == actors[1]["id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"../x.png",
|
||||
"x/y.png",
|
||||
"x\\y.png",
|
||||
"x\x00.png",
|
||||
"x\x7f.png",
|
||||
"x\n.png",
|
||||
"a" * 256,
|
||||
"",
|
||||
],
|
||||
)
|
||||
def test_filename_guard(client, actors, chat, name):
|
||||
check(upload(client, actors[0], chat, filename=name), 400, "Error")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data,mime,status",
|
||||
[
|
||||
(b"not an image", "image/png", 415),
|
||||
(PNG, "image/jpeg", 415),
|
||||
(PNG, "application/octet-stream", 415),
|
||||
(PNG[:25], "image/png", 415),
|
||||
(b"", "image/png", 415),
|
||||
(b"x" * (10 * 1024 * 1024 + 1), "image/png", 413),
|
||||
],
|
||||
)
|
||||
def test_image_guards(client, actors, chat, data, mime, status):
|
||||
r = upload(client, actors[0], chat, data=data, mime=mime)
|
||||
check(r, status, "Error")
|
||||
|
||||
|
||||
def test_jpeg_original_unchanged(client, actors, chat):
|
||||
stream = io.BytesIO()
|
||||
Image.new("RGB", (320, 200), (30, 60, 90)).save(stream, format="JPEG")
|
||||
jpeg = stream.getvalue()
|
||||
attachment = check(
|
||||
upload(
|
||||
client,
|
||||
actors[0],
|
||||
chat,
|
||||
data=jpeg,
|
||||
filename="camera.jpg",
|
||||
mime="image/jpeg",
|
||||
),
|
||||
202,
|
||||
"Attachment",
|
||||
)
|
||||
attachment = wait_ready(client, actors[0], attachment)
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{attachment['id']}/content",
|
||||
headers=headers(actors[1]),
|
||||
)
|
||||
assert r.content == jpeg
|
||||
|
||||
|
||||
def test_cross_chat_link_and_unknown(client, actors, chat):
|
||||
a = check(upload(client, actors[0], chat), 202, "Attachment")
|
||||
other = check(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "Other", "member_ids": [actors[0]["id"]]},
|
||||
),
|
||||
201,
|
||||
"Chat",
|
||||
)
|
||||
for identity in [a["id"], str(uuid4())]:
|
||||
check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{other['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": "", "attachment_ids": [identity]},
|
||||
),
|
||||
404,
|
||||
"Error",
|
||||
)
|
||||
check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": "", "attachment_ids": [a["id"], a["id"]]},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_default_attachments_are_semantically_equal(client, actors, chat):
|
||||
h = headers(actors[0])
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
first = check(
|
||||
client.post(path, headers=h, json={"text": "Semantic default"}),
|
||||
201,
|
||||
"Message",
|
||||
)
|
||||
second = check(
|
||||
client.post(
|
||||
path,
|
||||
headers=h,
|
||||
json={"attachment_ids": [], "text": "Semantic default"},
|
||||
),
|
||||
201,
|
||||
"Message",
|
||||
)
|
||||
assert first == second and first["attachment_ids"] == []
|
||||
|
||||
|
||||
def test_pixel_limit_before_decode(client, actors, chat):
|
||||
# Compressible synthetic image stays below10MiB but exceeds40M pixels.
|
||||
stream = io.BytesIO()
|
||||
Image.new("1", (8000, 5001)).save(stream, format="PNG")
|
||||
assert len(stream.getvalue()) < 10 * 1024 * 1024
|
||||
r = upload(client, actors[0], chat, data=stream.getvalue())
|
||||
assert r.status_code == 413
|
||||
|
||||
|
||||
def test_chunked_body_limit_without_content_length(client, actors, chat):
|
||||
r = client.post(
|
||||
f"/api/v1/chats/{chat['id']}/attachments",
|
||||
headers={**headers(actors[0]), "Content-Type": "image/png"},
|
||||
params={"filename": "chunked.png"},
|
||||
content=iter([b"x" * (10 * 1024 * 1024 + 1)]),
|
||||
)
|
||||
assert r.request.headers.get("content-length") is None
|
||||
assert r.status_code == 413
|
||||
assert r.json()["error"]["code"] == "payload_too_large"
|
||||
assert r.json()["error"]["request_id"] == r.headers["x-request-id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size", [(1, 17), (320, 200), (1600, 900)])
|
||||
def test_variant_geometry_and_alpha(client, actors, chat, size):
|
||||
stream = io.BytesIO()
|
||||
Image.new("RGBA", size, (0, 0, 0, 0)).save(stream, format="PNG")
|
||||
a = check(
|
||||
upload(client, actors[0], chat, data=stream.getvalue()),
|
||||
202,
|
||||
"Attachment",
|
||||
)
|
||||
a = wait_ready(client, actors[0], a)
|
||||
for v in a["variants"]:
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content",
|
||||
headers=headers(actors[1]),
|
||||
params={"variant": v["name"]},
|
||||
)
|
||||
assert len(r.content) == v["size_bytes"]
|
||||
assert hashlib.sha256(r.content).hexdigest() == v["sha256"]
|
||||
with Image.open(io.BytesIO(r.content)) as image:
|
||||
assert image.size == (v["width"], v["height"])
|
||||
assert not image.getexif()
|
||||
assert min(image.convert("RGB").getpixel((0, 0))) >= 250
|
||||
if v["name"] == "crop":
|
||||
assert image.size == (128, 128)
|
||||
else:
|
||||
bound = 256 if v["name"] == "thumbnail" else 1280
|
||||
assert max(image.size) <= bound
|
||||
assert image.width <= size[0] and image.height <= size[1]
|
||||
|
||||
|
||||
def test_exif_orientation_applied_but_original_preserved(client, actors, chat):
|
||||
stream = io.BytesIO()
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
exif[270] = "Synthetic metadata must be removed"
|
||||
Image.new("RGB", (320, 200), "red").save(stream, format="JPEG", exif=exif)
|
||||
raw = stream.getvalue()
|
||||
a = check(
|
||||
upload(
|
||||
client,
|
||||
actors[0],
|
||||
chat,
|
||||
data=raw,
|
||||
mime="image/jpeg",
|
||||
filename="exif.jpg",
|
||||
),
|
||||
202,
|
||||
"Attachment",
|
||||
)
|
||||
a = wait_ready(client, actors[0], a)
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=headers(actors[1])
|
||||
)
|
||||
assert r.content == raw
|
||||
v = next(v for v in a["variants"] if v["name"] == "thumbnail")
|
||||
assert (v["width"], v["height"]) == (160, 256)
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content",
|
||||
headers=headers(actors[1]),
|
||||
params={"variant": "thumbnail"},
|
||||
)
|
||||
with Image.open(io.BytesIO(r.content)) as image:
|
||||
assert image.size == (160, 256) and not image.getexif()
|
||||
@@ -0,0 +1,26 @@
|
||||
import json
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from conftest import check, headers
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["utf-16", "utf-32"])
|
||||
def test_only_utf8_json(client, encoding):
|
||||
payload = {"username": "u_" + uuid4().hex[:20], "display_name": "UTF"}
|
||||
payload["password"] = "Synthetic-encoding-test-123"
|
||||
result = client.post(
|
||||
"/api/v1/users",
|
||||
content=json.dumps(payload).encode(encoding),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
check(result, 400, "Error")
|
||||
|
||||
|
||||
def test_leading_zero_limit(client, actors, chat):
|
||||
result = client.get(
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
headers=headers(actors[0]),
|
||||
params={"limit": "0001"},
|
||||
)
|
||||
check(result, 200, "MessagePage")
|
||||
@@ -0,0 +1,23 @@
|
||||
from conftest import check, headers
|
||||
|
||||
|
||||
def test_history_acl_precedes_query_validation(client, actors, chat):
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
check(
|
||||
client.get(
|
||||
path,
|
||||
headers=headers(actors[2]),
|
||||
params={"limit": 0, "cursor": "broken"},
|
||||
),
|
||||
404,
|
||||
"Error",
|
||||
)
|
||||
check(
|
||||
client.get(
|
||||
path,
|
||||
headers=headers(actors[0]),
|
||||
params={"limit": 0, "cursor": "broken"},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
@@ -0,0 +1,75 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import BASE_URL, check, headers, history
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [None, "", "a b", "x" * 129, "a/b"])
|
||||
def test_key_required_and_valid(client, actors, chat, key):
|
||||
h = headers(actors[0])
|
||||
h.pop("Idempotency-Key")
|
||||
if key is not None:
|
||||
h["Idempotency-Key"] = key
|
||||
check(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers=h,
|
||||
json={"text": "Key"},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_parallel_replay_across_instances(client, actors, chat):
|
||||
h = headers(actors[0])
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
def send(_):
|
||||
with httpx.Client(base_url=BASE_URL, trust_env=False, timeout=15) as c:
|
||||
r = c.post(path, headers=h, json={"text": "Один результат 🌍"})
|
||||
return check(r, 201, "Message"), r.headers["x-instance-id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
results = list(pool.map(send, range(30)))
|
||||
assert len({r[0]["id"] for r in results}) == 1
|
||||
assert len({r[1] for r in results}) >= 2
|
||||
assert history(client, actors[0], chat["id"]) == [results[0][0]]
|
||||
check(
|
||||
client.post(path, headers=h, json={"text": "Другой текст"}),
|
||||
409,
|
||||
"Error",
|
||||
)
|
||||
# Другой пользователь может независимо использовать тот же ключ.
|
||||
other = headers(actors[1])
|
||||
other["Idempotency-Key"] = h["Idempotency-Key"]
|
||||
second = check(
|
||||
client.post(path, headers=other, json={"text": "Боб"}), 201, "Message"
|
||||
)
|
||||
assert second["id"] != results[0][0]["id"]
|
||||
|
||||
|
||||
def test_replay_rechecks_acl(client, actors, chat):
|
||||
h = headers(actors[0])
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
check(
|
||||
client.post(path, headers=h, json={"text": "Existing"}), 201, "Message"
|
||||
)
|
||||
outsider = headers(actors[2])
|
||||
outsider["Idempotency-Key"] = h["Idempotency-Key"]
|
||||
check(
|
||||
client.post(path, headers=outsider, json={"text": "Existing"}),
|
||||
404,
|
||||
"Error",
|
||||
)
|
||||
check(
|
||||
client.post(
|
||||
path,
|
||||
headers={"Idempotency-Key": uuid4().hex},
|
||||
json={"text": "Existing"},
|
||||
),
|
||||
401,
|
||||
"Error",
|
||||
)
|
||||
@@ -0,0 +1,114 @@
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import PASSWORD, TOKENS, check, headers, register
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"password", ["", "x" * 11, "x" * 129, " " * 12, 42, True, None]
|
||||
)
|
||||
def test_password_validation(client, password):
|
||||
check(
|
||||
client.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"username": "pwd_" + uuid4().hex[:20],
|
||||
"display_name": "Test",
|
||||
"password": password,
|
||||
},
|
||||
),
|
||||
400,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_full_utf8_password(client):
|
||||
password = "🌍" * 127 + ":"
|
||||
user = register(client, password=password)
|
||||
assert (
|
||||
check(
|
||||
client.get("/api/v1/users/me", headers=headers(user)), 200, "User"
|
||||
)
|
||||
== user
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"authorization", [None, "Basic broken", "Bearer unknown", "Basic /w=="]
|
||||
)
|
||||
def test_bad_basic(client, authorization):
|
||||
result = client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
headers={"Authorization": authorization} if authorization else {},
|
||||
)
|
||||
check(result, 401, "Error")
|
||||
assert (
|
||||
result.headers["www-authenticate"]
|
||||
== 'Basic realm="messenger", charset="UTF-8"'
|
||||
)
|
||||
|
||||
|
||||
def test_wrong_password_and_unknown_user(client):
|
||||
user = register(client)
|
||||
for username in [user["username"], "missing_" + uuid4().hex[:20]]:
|
||||
result = client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
auth=httpx.BasicAuth(username, PASSWORD + "wrong"),
|
||||
)
|
||||
data = check(result, 401, "Error")
|
||||
assert data["error"]["message"] == "Authentication required"
|
||||
|
||||
|
||||
def test_logout_is_not_revived_by_cache(client):
|
||||
user = register(client)
|
||||
auth = headers(user)
|
||||
check(client.get("/api/v1/users/me", headers=auth), 200, "User")
|
||||
result = client.delete("/api/v1/auth/sessions/current", headers=auth)
|
||||
assert result.status_code == 204 and result.content == b""
|
||||
assert result.headers["x-request-id"]
|
||||
check(client.get("/api/v1/users/me", headers=auth), 401, "Error")
|
||||
check(
|
||||
client.delete("/api/v1/auth/sessions/current", headers=auth),
|
||||
401,
|
||||
"Error",
|
||||
)
|
||||
|
||||
|
||||
def test_only_current_session_revoked(client):
|
||||
user = register(client)
|
||||
another = check(
|
||||
client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
auth=httpx.BasicAuth(user["username"], PASSWORD),
|
||||
),
|
||||
201,
|
||||
"Session",
|
||||
)
|
||||
assert another["token"] != TOKENS[user["id"]]
|
||||
assert len(another["token"]) >= 32
|
||||
assert (
|
||||
client.delete(
|
||||
"/api/v1/auth/sessions/current", headers=headers(user)
|
||||
).status_code
|
||||
== 204
|
||||
)
|
||||
result = client.get(
|
||||
"/api/v1/users/me",
|
||||
headers={"Authorization": "Bearer " + another["token"]},
|
||||
)
|
||||
assert check(result, 200, "User") == user
|
||||
|
||||
|
||||
def test_bearer_source_and_ignored_user_header(client, actors):
|
||||
alice, bob, _ = actors
|
||||
auth = {**headers(alice), "X-User-Id": bob["id"]}
|
||||
assert (
|
||||
check(client.get("/api/v1/users/me", headers=auth), 200, "User")["id"]
|
||||
== alice["id"]
|
||||
)
|
||||
result = client.get(
|
||||
"/api/v1/users/me", auth=httpx.BasicAuth(alice["username"], PASSWORD)
|
||||
)
|
||||
check(result, 401, "Error")
|
||||
assert result.headers["www-authenticate"] == 'Bearer realm="messenger"'
|
||||
@@ -0,0 +1,119 @@
|
||||
import copy
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator, FormatChecker
|
||||
|
||||
from src.pipeline import validate_job
|
||||
|
||||
|
||||
def example():
|
||||
return dict(
|
||||
schema_version=1,
|
||||
job_id=str(uuid4()),
|
||||
type="attachment.process.v1",
|
||||
attachment_id=str(uuid4()),
|
||||
pipeline_version=1,
|
||||
attempt=1,
|
||||
created_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
|
||||
|
||||
def test_schema_and_runtime_agree_on_valid_job():
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
schema = json.loads(Path("contracts/jobs.schema.json").read_text())
|
||||
for version in [1, 2]:
|
||||
data = example()
|
||||
data["pipeline_version"] = version
|
||||
data.update(
|
||||
traceparent="00-" + "a" * 32 + "-" + "b" * 16 + "-01",
|
||||
tracestate="vendor=opaque",
|
||||
)
|
||||
Draft202012Validator(schema, format_checker=FormatChecker()).validate(
|
||||
data
|
||||
)
|
||||
assert validate_job(data) == data
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("schema_version", True),
|
||||
("schema_version", 2),
|
||||
("type", "other"),
|
||||
("job_id", 123),
|
||||
("job_id", "broken"),
|
||||
("attachment_id", None),
|
||||
("pipeline_version", 0),
|
||||
("pipeline_version", True),
|
||||
("attempt", 0),
|
||||
("attempt", 1.0),
|
||||
("created_at", "2026-01-01"),
|
||||
("created_at", "2026-01-01T12:00:00"),
|
||||
("created_at", None),
|
||||
("traceparent", "not-w3c"),
|
||||
("tracestate", "x" * 513),
|
||||
("url", "http://invalid.example"),
|
||||
("bytes", "secret-contents"),
|
||||
("filename", "../../bad"),
|
||||
],
|
||||
)
|
||||
def test_invalid_notification_is_rejected(field, value):
|
||||
data = copy.deepcopy(example())
|
||||
data[field] = value
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
validate_job(data)
|
||||
|
||||
|
||||
def test_worker_rejects_oversized_pixels_before_decode():
|
||||
import hashlib
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from src.pipeline import PermanentFailure
|
||||
from src.worker import make_variants
|
||||
|
||||
stream = io.BytesIO()
|
||||
Image.new("1", (8000, 5001)).save(stream, format="PNG")
|
||||
raw = stream.getvalue()
|
||||
row = dict(
|
||||
size_bytes=len(raw),
|
||||
sha256=hashlib.sha256(raw).hexdigest(),
|
||||
content_type="image/png",
|
||||
pipeline_version=1,
|
||||
)
|
||||
with (
|
||||
pytest.warns(Image.DecompressionBombWarning),
|
||||
pytest.raises(PermanentFailure, match="pixel_limit"),
|
||||
):
|
||||
list(make_variants(raw, row))
|
||||
|
||||
|
||||
def test_worker_rejects_changed_source_checksum():
|
||||
from src.pipeline import PermanentFailure
|
||||
from src.worker import make_variants
|
||||
|
||||
with pytest.raises(PermanentFailure, match="source_checksum_mismatch"):
|
||||
list(make_variants(b"changed", dict(size_bytes=7, sha256="0" * 64)))
|
||||
|
||||
|
||||
def test_worker_rejects_invalid_image_as_permanent():
|
||||
import hashlib
|
||||
|
||||
from src.pipeline import PermanentFailure
|
||||
from src.worker import make_variants
|
||||
|
||||
raw = b"not an image"
|
||||
with pytest.raises(PermanentFailure, match="invalid_image"):
|
||||
list(
|
||||
make_variants(
|
||||
raw,
|
||||
dict(
|
||||
size_bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()
|
||||
),
|
||||
)
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user