ЛР 6 — БВТ2301 — Маслов — на 5 #64

Open
wh1tet3a wants to merge 13 commits from wh1tet3a/mtusi2026lab6:solution/bvt2301-maslov into main
111 changed files with 93526 additions and 51 deletions
+17
View File
@@ -18,3 +18,20 @@ __pycache__
node_modules
evidence
.DS_Store
**/__pycache__
**/*.py[cod]
.work
.pytest_cache
.ruff_cache
tests
student-tests
research
scripts
contracts
*.md
Makefile
compose.yaml
.work*
student-unit
+14 -9
View File
@@ -13,18 +13,23 @@ API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_BUCKET=messenger
S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key
S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
PUBLIC_PORT=8086
API_CPUS=0.5
API_MEMORY=768m
DB_POOL_SIZE=10
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_TIMEOUT_SECONDS=20
JOB_LEASE_SECONDS=23
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
OTEL_ENABLED=1
GRAFANA_PORT=3006
+19 -14
View File
@@ -1,18 +1,23 @@
## Лабораторная 6
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Маслов, БВТ2301.
Целевая оценка: 5. Трек 2 — SLO и обнаружение инцидентов.
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
К собственному приложению ЛР5 добавлена наблюдаемость API, relay, worker, отдельного процесса обработки, PostgreSQL и S3: OpenTelemetry, постоянные Prometheus/Loki/Tempo, provisioning Grafana на18панелей и правила оповещений по нескольким временным окнам. Метки ограничены; текст сообщений, имена файлов и credentials не экспортируются.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Полный чистый clone прошёл smoke,112HTTP,26unit, cluster/failover/rolling/persistence, все19сценариев отказов pipeline, очистку с реальным возрастом S3, причинные traces/logs, privacy и сохранность observability после рестарта. В обоих COMMIT→ACK окнах pending подтверждён до SIGKILL; после восстановления второй публикации нет. У7контейнеров приложения сверены source hashes и OCI revision.
Исследование1823.238с: все6370подтверждённых сообщений и359файлов сохранены; краткие отказы не вызывали critical paging (warning API был), длительные инциденты обнаружены и сняты; во время90с отказа Loki/Tempo прошли360message writes, очередь Collector≤64 и RSS<384MiB. CSV, сводки и графики приложены. Условия и погрешность замеров описаны в REPORT.
Ограничения: один host и одиночные stateful сервисы; учебные30mокна; цель99% намеренно нарушена контролируемыми сбоями;4неплановых503 в конце исследования сохранены в raw данных. Возможна потеря telemetry при отказе backend. SLO опыт выполнен на0.5CPU/API, финальная регрессия — на2CPU/API; разные профили явно указаны. ЛР6 использует локальный HTTP, TLS/mTLS вынесен в ЛР7.
Запуск: `uv sync --locked`, `make bootstrap`, `VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait`. API localhost8086, Grafana3006. Полные команды, результаты и ответы на4вопроса: REPORT.md. Финальная проверенная runtime-ревизия:231d965b1a10a26a845f66a9c148a5ce7ff07701.
Источники: собственная ЛР5, задание, официальная документация OTel/Grafana/Prometheus. Использована помощь ИИ при проектировании, реализации, проверках и подготовке отчёта; чужие студенческие решения не использовались.
- [x] Обязательный минимум подтверждён полной регрессией из чистого клона.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены; реальные receipts приложены.
- [x] Отказы observability/SLO выбранного трека воспроизведены.
- [x] Секреты и приватные ключи вне Git, значения не включены в отчёт.
- [x] README задания сохранён; REPORT описывает именно это решение.
+5
View File
@@ -30,3 +30,8 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
.work/
.work-*.txt
.pytest_cache/
.ruff_cache/
+22
View File
@@ -0,0 +1,22 @@
FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
COPY --from=uv /uv /usr/local/bin/uv
WORKDIR /app
ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy
COPY pyproject.toml uv.lock ./
RUN --mount=type=cache,target=/root/.cache/uv uv sync --locked --no-dev --no-install-project
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
ARG VCS_REF=unknown
LABEL org.opencontainers.image.revision=$VCS_REF
WORKDIR /app
ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
COPY --from=builder /app/.venv /app/.venv
COPY src /app/src
COPY migrations /app/migrations
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"]
STOPSIGNAL SIGTERM
CMD ["python", "-m", "src"]
+23
View File
@@ -28,3 +28,26 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
UV ?= uv
BASE_URL = http://127.0.0.1:8086
.PHONY: bootstrap up down lint test-student verify
bootstrap:
$(PYTHON) scripts/bootstrap.py
up: bootstrap
VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait
down:
docker compose down
lint:
$(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py $(wildcard scripts/verify_*.py)
$(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py $(wildcard scripts/verify_*.py)
test-student:
BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests
verify:
$(UV) run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json
.PHONY: unit reconcile
unit:
PYTHONPATH=. $(UV) run --locked pytest student-unit -q
reconcile:
docker compose run --rm --no-deps reconciler python -m src.reconcile
+103 -24
View File
@@ -1,37 +1,116 @@
# Отчёт — лабораторная 6
# Лабораторная 6 — Маслов, БВТ2301
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Цель — оценка 5, трек 2: SLO и обнаружение инцидентов. Собственный код ЛР5 перенесён поверх независимого skeleton ЛР6. Исходные README/COURSE/контракты/публичные tests и check/validate сохранены. Бизнес-API и устойчивый pipeline файлов продолжают работать; добавлена наблюдаемость всех стадий.
## Запуск из чистого clone
## Воспроизведение
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Desktop/Compose v2, Git и uv0.13.7. Python3.14.7, SDK OpenTelemetry1.45.2, Collector0.165.0, Prometheus3.16.0, Loki3.7.10, Tempo3.1.1, Grafana13.3.1. Версии и image digests закреплены. MinIO/mc собираются из точных upstream commits с GoSumDB.
## Архитектура и решения
```sh
uv sync --locked
make bootstrap
VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait
make check validate lint unit PYTHON=.venv/bin/python
make test BASE_URL=http://127.0.0.1:8086 PYTHON=.venv/bin/python
make test-student
uv run --locked python -m scripts.verify_telemetry --output evidence/telemetry-repeat.json
uv run --locked python -m scripts.verify_observability_persistence --causal-evidence evidence/telemetry-repeat.json --output evidence/persistence-repeat.json
uv run --locked python -m scripts.research_slo --output evidence/slo-repeat.json
uv run --locked python -m research.analyze_slo evidence/slo-repeat.json --output evidence/slo-analysis-repeat
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
API: http://127.0.0.1:8086. Grafana: http://127.0.0.1:3006, admin, пароль в локальном `secrets/grafana_password`. Bootstrap сохраняет прежние файлы и не печатает значения. Provisioning трёх datasources и dashboard выполняется автоматически. Внутренние OTLP/metrics/backend ports на host не публикуются. `docker compose down` сохраняет данные; secrets и volumes не входят в Git. Fault hooks включаются только явным `compose.test.yaml`.
## Доказательства критериев
Проверка чистого клона: `uv run --locked python scripts/verify_clean.py --output evidence/clean-clone-repeat.json`. Она создаёт собственные временные clone/venv/secrets/volumes/ports/image, выполняет полную регрессию и удаляет только временный проект. Основной стенд перед ней следует остановить, сохранив тома. Имена output должны быть новыми: предыдущие результаты не перезаписываются.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
## Архитектура и сигналы
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
API/relay/worker/observer → OTLP/HTTP → Collector → Prometheus/Loki/Tempo → запросы Grafana. Grafana показывает данные; сама она не заменяет хранилища трёх сигналов. Каждый backend и Grafana имеют отдельный persistent volume. API-реплики, PG, S3, broker, worker, сессии, ACL и идемпотентность сохранены.
## Эксперименты и отказы
Migration005 сохраняет W3C context в принимающей транзакции вместе с jobs/outbox. Relay создаёт producer span и передаёт его контекст в stream. Worker создаёт consumer span, передаёт carrier изолированному дочернему процессу; в trace видны обработка изображения, S3 и PostgreSQL. Повторная попытка сохраняет причинную связь. `traceparent` сохраняется при отбрасывании слишком длинного `tracestate`, чтобы не нарушать строгую схему jobs.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Логи структурные: event/service/trace/span/request/технический job ID. Экспорт полей задан allowlist. Тела сообщений, изображения, пароли, Authorization, SQL-тексты, raw URL и exception message/stack не экспортируются. Идентификаторы пользователей/чатов/сообщений не становятся labels. Метрики используют route template, ограниченный HTTP method/status class, job type/outcome, service instance/version.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Dashboard `messenger-lab6` содержит 18 панелей: RPS, 5xx, HTTP p50/p95/p99, jobs/duration, время до ready, очередь/возраст, worker busy/capacity, DB pool used/waiting, память/CPU и Collector. Percentile в dashboard вычисляется `histogram_quantile` по сумме bucket rates с сохранённым le; значения разных реплик сначала объединяются. Определения и единицы находятся в импортируемом JSON. HTTP SDK duration измеряет ASGI response; end-to-end клиентские quantiles в исследовании — отдельные величины, вычисленные nearest rank по успешным HTTP запросам.
## Ограничения и следующие шаги
SDK traces/logs: очередь 512 записей, batch64, конечные таймауты. Collector: memory limiter192MiB/spike32, batch≤256, очереди logs/traces по32 пакета, один отправитель, конечные повторы до10с, nonblocking overflow. Collector CPU0.5/RAM384MiB, Tempo CPU0.5/RAM768MiB. Потери telemetry при длительном отказе допустимы и наблюдаемы; не заявляется надёжная доставка каждого span.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
## SLI/SLO и гипотеза
Две учебные цели 99% за окно30мин: ответы business ingress без5xx и готовность принятого файла не позднее10с. API SLI берётся с HAProxy business backend, включая ingress503; health использует отдельный backend. Ready SLI сравнивает PG created_at принимающей транзакции с committed_at публикации результата. Начальная граница консервативно включает завершение принимающей транзакции.
Незавершённые после10с и failed учитываются как bad, а не исчезают из знаменателя. Решение сохраняется однократно в PG; replay не стирает прежний bad, obsolete до решения исключён. Histogram времени готовности содержит только завершения и не заменяет этот SLI.
Гипотеза: сочетание короткого и длинного окна в burn-rate rule отсечёт краткий всплеск от критического оповещения и обнаружит длительное нарушение обоих SLO. Burn rate — доля плохих событий/0.01. Fast: >14.4 одновременно в30с и2мин, for45с. Slow: >6 в2мин и5мин, for60с. Дополнительно warning ApiHighErrorRate и QueueLagGrowing. Это ускоренный учебный профиль; значения не выдаются за универсальную production policy.
## Методика и результаты исследования
Полный успешный опыт — `evidence/slo-research-verified.json`, CSV запросов и `evidence/slo-analysis-verified/summary.json`, таблица фаз и [график](evidence/slo-analysis-verified/slo-timeline.png). Длительность1823.238с. Нагрузка: открытое планирование4 message POST/с, максимум16 in-flight; PNG320×200 каждые5с. Docker VM:8CPU и фактический объём памяти в receipt; все service quotas сохранены там же.
Опыт проводился при API0.5CPU/768MiB на реплику (точные limits в receipt). Финальный default API увеличен до1CPU после отдельной проверки login latency; это не тот же ресурсный профиль и результаты не выдаются за повторное измерение на1CPU. Для повторения именно опыта сначала развернуть `COMPOSE_FILE=compose.yaml:compose.test.yaml API_CPUS=0.5 VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait`, затем запускать research с теми же COMPOSE_FILE/API_CPUS.
После разогрева300с воспроизводятся короткие и длительные сбои. Для API — задержка SQL INSERT только синтетического чата, превышающая statement_timeout. Для ready — остановка worker. Затем оба telemetry хранилища Loki/Tempo останавливаются на90с, при этом сообщения и задания продолжают приниматься. После восстановления найдены свежие trace и log. В конце прекращается нагрузка и через публичный API проверяются все подтверждённые данные.
| Длительный инцидент / оповещение | Обнаружение, с после действия | Снятие, с после восстановления компонента |
|---|---:|---:|
| API / FastBurn |73.368|39.261|
| API / SlowBurn |84.651|127.152|
| API / HighErrorRate warning |44.859|127.152|
| Worker / QueueLag warning |47.281|181.174|
| Worker / FastBurn |76.710|235.563|
| Worker / SlowBurn |85.981|303.950|
При коротких сбоях критических paging событий0. Короткий API сбой всё же вызвал warning ApiHighErrorRate во время восстановления: не утверждается отсутствие всех оповещений. Очередь и окна не очищаются мгновенно после запуска worker; время снятия включает дренирование и исчезновение старых нарушений из окон.
В ходе90с отказа telemetry сохранены360 успешных message POST, Collector queue достигла64 при суммарной ёмкости64. Его RSS в опыте238.918–239.035MiB при лимите384MiB. После восстановления очередь0, новая trace/log доступны. Счётчик неуспешных отправок показывает попытки, а не уникально потерянные spans; точное количество потерянных spans не выводится из него.
В конце доступны все6370 подтверждённых сообщений, все359 принятых файлов находятся в ready; проверенные исходники побайтно совпадают. Это проверка сохранности подтверждённых операций, а не утверждение, что все запросы были успешны. Во время преднамеренных сбоев SLI API89.925%, ready79.832%; цель99% специально нарушена, что и должны обнаружить правила. В заключительной фазе также зарегистрированы4 ответа503; они не скрываются из CSV или расчётов.
Частота scrape/evaluation5с и экспорт ограничивают разрешение, а реальный polling наблюдателя дал промежутки5.111–38.223с, медиана9.663с. Три десятичных знака — арифметика временных меток, не миллисекундная точность обнаружения. Клиентские фазовые сводки используют start timestamp внутри фактически измеренного hold; промежутки orchestration исключены, raw CSV сохранён.
## Проверка и разбор инцидентов
Полный повтор из чистого клона завершился PASS: [clean-clone-final.json](evidence/clean-clone-final.json). Проверенная ревизия приложения и конфигурации — `231d965b1a10a26a845f66a9c148a5ce7ff07701`; дальнейшие изменения относятся к документации и результатам.
| Проверка | Подтверждённый результат |
|---|---|
| Структура, полный OpenAPI, lint, unit | PASS, 26 unit-тестов |
| Публичный smoke и собственная HTTP-регрессия | PASS, 112 HTTP-тестов за556.50с |
| Общая идемпотентность, rollback, отказ каждой API, rolling replacement | PASS, подтверждённые сообщения сохранены |
| Исходные публичные failover/persistence | PASS; failover120успешных writes/5неуспешных попыток, recovery3.206с≤10с; прежняя сессия, сообщения и файл сохранены |
| Отказы pipeline | Все19сценариев PASS, [полный результат](evidence/pipeline-final.json), [таблица](evidence/pipeline-analysis-final/effects.csv), [график](evidence/pipeline-analysis-final/pipeline-results.png) |
| COMMIT перед ACK | В обоих повторах ready/PUB1/4файла и реальные pending до SIGKILL, child_finished отсутствует; после восстановления подтверждение очереди без второй публикации |
| Старая версия после новой | Два повтора: v2 сохранена, устаревшая v1 не опубликована |
| Очистка | Реальный возраст S3≥1051.823с при retention300с; dry-run не меняет данные, удалены10устаревших объектов, повтор0действий; current/active/fresh сохранены; replay без исходника отклонён |
| Telemetry и provisioning | Причинные normal/retry traces и связанные logs, privacy canaries, низкая cardinality, 18Grafana panels и реальные datasource queries — PASS |
| Persistence observability | Перезапуск четырёх хранилищ: прежние metrics/trace/log и настройка Grafana сохранены |
| Запущенный код | SHA-256 всех Python-файлов и OCI revision совпали с проверенной ревизией у2API/worker/relay/observer/reconciler/controller; fsck PASS |
Все проверки выполнялись на новых secrets, venv, образе, портах и named volumes. Временный проект после проверки удалён; основные тома сохранены. Контракт и публичные tests не ослаблялись. `telemetry-first.json` и SLO исследование содержат также отдельные исходные измерения; результаты ЛР5 не выдаются за измерения ЛР6.
Итоговый профиль API —2CPU/768MiB на реплику, Argon2id64MiB/t3/p4 и2KDF slots. Отдельный [CPU benchmark](evidence/password-cpu-profile.json) дал при1CPU медиану0.30070с/максимум0.66556с, при2CPU0.13229/0.19519с на30hashes. Это измерение отдельного KDF, не HTTP latency. Сам бенчмарк timeout2с не воспроизвёл; рабочий профиль подтверждён полным прогоном выше. Исторический SLO опыт выполнен на0.5CPU/768MiB и исходниках752a5e6; его результаты не выдаются за измерения финального профиля.
Неуспешные попытки сохранены и не используются как PASS:
| Receipt | Причина остановки |
|---|---|
| clean-clone-final-failed.json | Unhealthy API до HTTP; точная причина без container logs не установлена |
| clean-clone-final-failed-af292f62.json | После112HTTP/telemetry/persistence вход занял2.12489с при public timeout2с и квоте0.5CPU |
| clean-clone-final-failed-8ffd4695.json | Наблюдатель пропустил узкое COMMIT→ACK окно; переделан на заранее запущенный процесс, worker timeout/lease сохранены |
| clean-clone-final-failed-04082c1f.json | Конкурирующий стенд; массовые503 и задержки,49PASS/19FAIL/44ERROR |
| clean-clone-final-failed-708b4c7d.json |112HTTP/telemetry/persistence прошли, но регистрация public resilience не уложилась в2с при1CPU; точное место задержки по access tail не установлено |
Повторный verifier сначала проверяет smoke/cluster/pipeline/GC, затем112HTTP/telemetry/persistence и7runtime hashes. Проверки сохранены целиком; такой порядок раньше обнаруживает сбои узких окон. Разбор самого SLO инцидента и действия оператора: [RUNBOOK](infra/otel/RUNBOOK.md).
## Ответы на контрольные вопросы
1. Метрика показывает масштаб/время проблемы: например доля5xx или возраст очереди. Лог — конкретное событие и технические ID. Trace — путь одного запроса/задания и длительности отдельных стадий. Вместе они позволяют перейти от общей деградации к причине.
2. Каждый новый chat_id создаёт отдельный временной ряд для каждой комбинации labels. Рост чатов умножает число рядов и стоимость RAM/индекса/запросов. Поэтому chat_id оставляют в техническом логе/trace, а метрику группируют по ограниченным признакам.
3. Контекст traceparent/tracestate сохраняется вместе с заданием, relay создаёт producer span и записывает его carrier в уведомление. Worker извлекает его и создаёт consumer span с правильным parent; передача в child не теряется при смене процесса. Durable outbox отделяет сохранение задания от доставки уведомления.
4. Среднее скрывает редкие долгие ответы; p99 описывает хвост. Усреднять p99 реплик нельзя: разные объёмы запросов и распределения не восстанавливаются из двух quantiles. Нужно сложить совместимые histogram buckets реплик и вычислить общий percentile.
## Источники и ограничения
Задание, собственная ЛР5 и официальные материалы: [OTLP Python](https://opentelemetry.io/docs/languages/python/exporters/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [Tempo3.1.1 config](https://github.com/grafana/tempo/blob/v3.1.1/example/docker-compose/single-binary/tempo.yaml), [Collector](https://opentelemetry.io/docs/collector/configuration/), [Prometheus alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/). При проектировании, реализации, проверках и подготовке отчёта использована помощь ИИ. Чужие студенческие решения/истории/измерения не использовались.
Один host и одиночные PG/S3/broker/LB остаются SPOF; нет защиты от потери диска. SLO исследован один раз полностью на данном профиле; статистическая обобщаемость не заявляется. Остались явные потери запросов/telemetry при искусственных отказах; они отражены в raw receipts. HTTP используется только локально с синтетическими данными; TLS/mTLS вводятся независимо в ЛР7.
+30
View File
@@ -0,0 +1,30 @@
# Explicit test-only fault instrumentation, never enabled by default.
services:
api:
environment:
PIPELINE_FAULTS: "1"
relay:
environment:
PIPELINE_FAULTS: "1"
worker:
environment:
PIPELINE_FAULTS: "1"
JOB_TIMEOUT_SECONDS: "12"
JOB_LEASE_SECONDS: "15"
controller:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command: [python, -c, "import time; time.sleep(86400)"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
secrets: [database_url, broker_url, s3_access_key, s3_secret_key]
cpus: 0.5
mem_limit: 192m
memswap_limit: 192m
read_only: true
healthcheck: {disable: true}
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
+508
View File
@@ -0,0 +1,508 @@
name: mtusi-lab6-maslov
services:
postgres:
image: postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
PGDATA: /var/lib/postgresql/data/pgdata
secrets:
- postgres_password
volumes:
- postgres_data:/var/lib/postgresql
healthcheck:
test:
- CMD-SHELL
- pg_isready -U messenger -d messenger
interval: 2s
timeout: 2s
retries: 20
cpus: 1
mem_limit: 512m
memswap_limit: 512m
cache:
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command:
- valkey-server
- /run/secrets/valkey.conf
secrets:
- cache_password
- source: valkey_config
target: valkey.conf
healthcheck:
test:
- CMD-SHELL
- REDISCLI_AUTH="$$(cat /run/secrets/cache_password)" valkey-cli ping
interval: 2s
timeout: 2s
retries: 20
cpus: 0.25
mem_limit: 128m
memswap_limit: 128m
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command:
- python
- -m
- src.migrate
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
cpus: 0.5
mem_limit: 256m
read_only: true
api:
image: ${API_IMAGE:-mtusi-messenger:lab6}
build:
context: .
args:
VCS_REF: ${VCS_REF:-unknown}
scale: ${API_REPLICAS:-2}
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_SIZE: ${DB_POOL_SIZE:-10}
OTEL_ENABLED: ${OTEL_ENABLED:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
APP_VERSION: ${VCS_REF:-development}
secrets:
- database_url
- cache_url
- s3_access_key
- s3_secret_key
depends_on:
migrate:
condition: service_completed_successfully
cache:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
cpus: ${API_CPUS:-2}
mem_limit: ${API_MEMORY:-768m}
memswap_limit: ${API_MEMORY:-768m}
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).read()"]
interval: 3s
timeout: 3s
start_period: 30s
retries: 12
pids_limit: 128
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=128m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
stop_grace_period: 10s
s3:
image: ${S3_IMAGE:-mtusi-storage:minio-20251015}
build:
context: .
dockerfile: infra/s3.Dockerfile
target: server
secrets:
- s3_root_user
- s3_root_password
volumes:
- s3_data:/data
cpus: 1
mem_limit: 1024m
memswap_limit: 1024m
read_only: true
tmpfs:
- /tmp:rw,nosuid,size=32m
environment:
HOME: /tmp
security_opt:
- no-new-privileges:true
s3-bootstrap:
image: ${MC_IMAGE:-mtusi-storage:mc-20250813}
build:
context: .
dockerfile: infra/s3.Dockerfile
target: bootstrap
secrets:
- s3_root_user
- s3_root_password
- s3_access_key
- s3_secret_key
depends_on:
s3:
condition: service_healthy
cpus: 0.25
mem_limit: 128m
read_only: true
tmpfs:
- /tmp:rw,nosuid,size=16m
reconciler:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command:
- python
- -m
- src.reconcile
- --apply
- --loop
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
secrets:
- database_url
- s3_access_key
- s3_secret_key
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
cpus: 0.25
mem_limit: 256m
memswap_limit: 256m
read_only: true
healthcheck:
disable: true
broker:
image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
command:
- valkey-server
- /run/secrets/broker.conf
secrets:
- broker_password
- source: broker_config
target: broker.conf
volumes:
- broker_data:/data
healthcheck:
test:
- CMD-SHELL
- REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping
interval: 2s
timeout: 2s
retries: 20
cpus: 0.5
mem_limit: 256m
memswap_limit: 256m
relay:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command:
- python
- -m
- src.pipeline
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
OTEL_ENABLED: ${OTEL_ENABLED:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
APP_VERSION: ${VCS_REF:-development}
secrets:
- database_url
- broker_url
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
cpus: 0.25
mem_limit: 192m
memswap_limit: 192m
read_only: true
healthcheck:
disable: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
worker:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command:
- python
- -m
- src.worker
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-23}
OTEL_ENABLED: ${OTEL_ENABLED:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
APP_VERSION: ${VCS_REF:-development}
secrets:
- database_url
- broker_url
- s3_access_key
- s3_secret_key
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
cpus: 2
mem_limit: 1536m
memswap_limit: 1536m
pids_limit: 128
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=64m
healthcheck:
disable: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
stop_grace_period: 25s
lb:
image: haproxy:3.2.1-alpine@sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9
ports:
- 127.0.0.1:${PUBLIC_PORT:-8086}:8080
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
read_only: true
tmpfs:
- /tmp:rw,nosuid,size=4m
security_opt:
- no-new-privileges:true
stop_grace_period: 10s
observer:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
command:
- python
- -m
- src.observer
environment:
DATABASE_URL_FILE: /run/secrets/database_url
OTEL_ENABLED: ${OTEL_ENABLED:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
APP_VERSION: ${VCS_REF:-development}
secrets:
- database_url
depends_on:
migrate:
condition: service_completed_successfully
cpus: 0.25
mem_limit: 192m
memswap_limit: 192m
read_only: true
healthcheck:
disable: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
otel:
image: otel/opentelemetry-collector-contrib:0.165.0@sha256:e40d61f10c9a617cca1ffd1be4d7e2b3a4b0f0001798ff07e58cabe82e618cea
user: 10001:10001
volumes:
- ./infra/otel/collector.yaml:/etc/otel/config.yaml:ro
cpus: 0.5
mem_limit: 384m
memswap_limit: 384m
environment:
GOMEMLIMIT: 224MiB
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
command:
- --config=/etc/otel/config.yaml
prometheus:
image: prom/prometheus:v3.16.0@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06
user: 65534:65534
volumes:
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
cpus: 0.5
mem_limit: 384m
memswap_limit: 384m
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
command:
- --config.file=/etc/prometheus/prometheus.yaml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=24h
- --storage.tsdb.retention.size=256MB
depends_on:
telemetry-init:
condition: service_completed_successfully
loki:
image: grafana/loki:3.7.10@sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7
user: 10001:10001
volumes:
- ./infra/otel/loki.yaml:/etc/loki/config.yaml:ro
- loki_data:/loki
cpus: 0.5
mem_limit: 384m
memswap_limit: 384m
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
command:
- -config.file=/etc/loki/config.yaml
depends_on:
telemetry-init:
condition: service_completed_successfully
tempo:
image: grafana/tempo:3.1.1@sha256:64e2fbd56d9c2b3412a4b430a58ea59e91a670b1e75f920a0c702213ad145044
user: 10001:10001
volumes:
- ./infra/otel/tempo.yaml:/etc/tempo/config.yaml:ro
- tempo_data:/var/tempo
cpus: 0.5
mem_limit: 768m
memswap_limit: 768m
environment:
GOMEMLIMIT: 384MiB
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
command:
- -config.file=/etc/tempo/config.yaml
depends_on:
telemetry-init:
condition: service_completed_successfully
grafana:
image: grafana/grafana:13.3.1@sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf
user: 472:472
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
cpus: 0.5
mem_limit: 384m
memswap_limit: 384m
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
GF_AUTH_ANONYMOUS_ENABLED: 'false'
GF_USERS_ALLOW_SIGN_UP: 'false'
GF_ANALYTICS_REPORTING_ENABLED: 'false'
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: 'false'
GF_PLUGINS_PREINSTALL_DISABLED: 'true'
GF_LOG_LEVEL: warn
GOMEMLIMIT: 256MiB
secrets:
- grafana_password
ports:
- 127.0.0.1:${GRAFANA_PORT:-3006}:3000
depends_on:
telemetry-init:
condition: service_completed_successfully
telemetry-init:
image: ${API_IMAGE:-mtusi-messenger:lab6}
pull_policy: never
user: 0:0
command:
- python
- -c
- import os; [(os.makedirs(p,exist_ok=True),os.chown(p,u,u)) for p,u in [('/data/prometheus',65534),('/data/loki',10001),('/data/tempo',10001),('/data/grafana',472)]]
volumes:
- prometheus_data:/data/prometheus
- loki_data:/data/loki
- tempo_data:/data/tempo
- grafana_data:/data/grafana
read_only: true
cap_drop:
- ALL
cap_add:
- CHOWN
security_opt:
- no-new-privileges:true
cpus: 0.1
mem_limit: 64m
healthcheck:
disable: true
secrets:
broker_password:
file: ./secrets/broker_password
broker_url:
file: ./secrets/broker_url
broker_config:
file: ./secrets/broker.conf
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key:
file: ./secrets/s3_access_key
s3_secret_key:
file: ./secrets/s3_secret_key
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_password:
file: ./secrets/cache_password
cache_url:
file: ./secrets/cache_url
valkey_config:
file: ./secrets/valkey.conf
name: valkey.conf
grafana_password:
file: ./secrets/grafana_password
volumes:
broker_data: null
s3_data: null
postgres_data: null
prometheus_data: null
loki_data: null
tempo_data: null
grafana_data: null
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+11
View File
@@ -0,0 +1,11 @@
{
"date": "2026-11-20T15:15:34.500702+00:00",
"source_commit": "2c7b89bb8e68ac79971b3314a8f7f97eb30f0cf5",
"fresh_secrets_environment_and_volumes": true,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": 57033,
"static_checks": ".venv/bin/python scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n.venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\n<home>/.local/bin/uv run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py scripts/verify_clean.py scripts/verify_cleanup.py scripts/verify_cluster.py scripts/verify_observability_persistence.py scripts/verify_pipeline.py scripts/verify_telemetry.py\n46 files already formatted\n<home>/.local/bin/uv run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py scripts/verify_clean.py scripts/verify_cleanup.py scripts/verify_cluster.py scripts/verify_observability_persistence.py scripts/verify_pipeline.py scripts/verify_telemetry.py\nAll checks passed!",
"unit_tests": "PYTHONPATH=. <home>/.local/bin/uv run --locked pytest student-unit -q\n.......................... [100%]\n26 passed in 10.87s",
"failed_action": "docker compose up -d --build --wait",
"result": "INCOMPLETE"
}
File diff suppressed because it is too large Load Diff
+60
View File
@@ -0,0 +1,60 @@
{
"source_commit": "a19e8053a7b22f067c2ac7a60956adecacc275a2",
"api_cpus": 1,
"client_timeout_seconds": 2,
"complete": true,
"login_trials": [
{
"register_seconds": 0.5860111659858376,
"login_seconds": 0.5677100410684943,
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
},
{
"register_seconds": 0.5128639999311417,
"login_seconds": 0.3723337920382619,
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
},
{
"register_seconds": 0.35827833297662437,
"login_seconds": 0.6672967500053346,
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
},
{
"register_seconds": 0.4009002910461277,
"login_seconds": 0.577231707982719,
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
},
{
"register_seconds": 0.346562874969095,
"login_seconds": 0.2660798749420792,
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
},
{
"register_seconds": 0.4216097079915926,
"login_seconds": 0.3548308340832591,
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
},
{
"register_seconds": 0.3752310830168426,
"login_seconds": 0.45202583307400346,
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
},
{
"register_seconds": 0.5370608329540119,
"login_seconds": 0.34091954200994223,
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
},
{
"register_seconds": 0.35265858308412135,
"login_seconds": 0.46283045900054276,
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
},
{
"register_seconds": 0.44550816691480577,
"login_seconds": 0.4593870830722153,
"instance": "04540f4e28ac4abeae99b4bf8e1ec0ad"
}
],
"failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 82, \"failed_attempts\": 6, \"first_recovery_seconds\": 9.174}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
"persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
}
+8
View File
@@ -0,0 +1,8 @@
{
"source_commit": "8f3ff33b2dcb1cf3082535acf99ee4045a80400d",
"command": "BASE_URL=http://127.0.0.1:8086 uv run --locked pytest student-tests -q",
"elapsed_seconds": 417.533,
"exit_code": 0,
"summary": "112 passed in 416.52s (0:06:56)",
"result": "PASS"
}
+130
View File
@@ -0,0 +1,130 @@
{
"scope": "isolated Argon2 library benchmark, not end-to-end HTTP latency",
"same_memory_time_parameters": true,
"one_cpu": {
"cpus": 1,
"memory_limit_bytes": 268435456,
"memory_cost_kib": 65536,
"time_cost": 3,
"alternating_order": true,
"samples": {
"1": [
0.583163583998612,
0.4382456249986717,
0.2641096250008559,
0.19429424999907496,
0.6078042919998552,
0.24880558400036534,
0.2602112919994397,
0.46108737500071584,
0.2328668749996723,
0.1995170419995702,
0.6032528750001802,
0.4759230830004526,
0.29089025000030233,
0.20416029199986951,
0.20563058300103876,
0.1919123749994469,
0.28446004199940944,
0.19227595799929986,
0.2107001250005851,
0.19277724999847123,
0.2145693759994174,
0.1922177920005197,
0.21456558399950154,
0.19194708300165075,
0.2767800420006097,
0.19193233300029533,
0.28086145800079976,
0.19231987499915704,
0.2498207500011631,
0.19609583399869734
],
"4": [
0.3964598330003355,
0.6655551250005374,
0.6048233330002404,
0.24411720799980685,
0.606782042001214,
0.22436758300136717,
0.3119742920007411,
0.39041379099944606,
0.579897167001036,
0.2679342920000636,
0.515006749999884,
0.41912158299965085,
0.5016956670006039,
0.29056270900036907,
0.2986015829992539,
0.22049533300014446,
0.303171375000602,
0.20446145800087834,
0.3835568750000675,
0.2309797920006531,
0.3757739169996057,
0.2987268329998187,
0.29915149999942514,
0.2104414170007658,
0.3022428750009567,
0.22631599999840546,
0.3057695839997905,
0.220404542000324,
0.2900597080006264,
0.25032029099929787
]
},
"summary": {
"1": {
"median_seconds": 0.22371812549954484,
"max_seconds": 0.6078042919998552
},
"4": {
"median_seconds": 0.3006971875001909,
"max_seconds": 0.6655551250005374
}
},
"started_at": 1795193933.5921836
},
"two_cpu_same_parallelism": {
"cpus": 2,
"memory_limit_bytes": 268435456,
"memory_cost_kib": 65536,
"time_cost": 3,
"parallelism": 4,
"samples": [
0.11252029100069194,
0.11649812500036205,
0.10306574999958684,
0.1575070830003824,
0.1271223750009085,
0.1904467090007529,
0.1951932090014452,
0.1331768749987532,
0.18574516700027743,
0.18905379200077732,
0.19207829200058768,
0.12781625000025088,
0.15630154199971003,
0.12129762500080687,
0.10808220800026902,
0.11075858399999561,
0.10086133400000108,
0.17311095900004148,
0.10017191600127262,
0.14115979200141737,
0.17558462499982852,
0.1898825839998608,
0.18430233300023247,
0.13139983299879532,
0.19080974999997125,
0.11175649999859161,
0.10798150000118767,
0.16109262500140176,
0.10722962499858113,
0.11008891700112144
],
"median_seconds": 0.13228835399877426,
"max_seconds": 0.1951932090014452
},
"decision": "Retain Argon2id m=65536KiB t=3 p=4; reserve 2 CPU for each API replica. Full HTTP verification still required."
}
@@ -0,0 +1,20 @@
case,role,job_id,version,attempt,cycle,fence,state,publications,variant_rows,physical_keys
commit_before_publish_and_worker_stopped,effect,cc13ff33-a32a-405f-9f9e-9586978137a1,1,1,1,1,ready,1,4,4
confirmed_broker_entry_survives_sigkill,effect,0f014c3c-ab27-4f88-b195-043f23718200,1,1,1,1,ready,1,4,4
relay_crash_after_xadd_before_pg_marker,effect,c61da309-5d2c-4beb-958e-a2ca62f9939d,1,1,1,1,ready,1,4,4
before_effect_sigkill_repeat_1,effect,5f3097f7-410e-46ee-a05d-ee44f82b79dc,1,2,1,3,ready,1,4,4
after_effect_sigkill_repeat_1,effect,27d8895e-aa28-430d-98c4-6b2344140fd1,1,2,1,3,ready,1,4,4
after_commit_sigkill_repeat_1,effect,069ea9cb-342f-4339-80d7-4cfb2c5b0e21,1,1,1,1,ready,1,4,4
before_effect_sigkill_repeat_2,effect,11f50310-4da8-4404-9ac5-86010952cd92,1,2,1,3,ready,1,4,4
after_effect_sigkill_repeat_2,effect,3f39caa5-e08f-4716-92c2-ca31eea0004b,1,2,1,3,ready,1,4,4
after_commit_sigkill_repeat_2,effect,47446e4a-c8cb-41e3-b5e0-0da62d3c699c,1,1,1,1,ready,1,4,4
transient_retry_with_jitter,effect,31bed0a2-2169-4b6d-8b44-ec63a1cfb448,1,2,1,3,ready,1,4,4
hung_child_killed_and_retried,effect,8ee47c13-e328-43b4-aae8-f5f64d8d96dc,1,2,1,3,ready,1,4,4
permanent_failed_registry_and_audited_replay,effect,9d7ed4df-7397-450f-ba54-4f1848525a8f,1,1,2,4,ready,1,4,4
finite_retry_exhaustion_then_explicit_replay,effect,3a3d7792-9772-4a40-8eff-9db1515f60ce,1,1,2,12,ready,1,4,4
reordered_and_duplicate_notifications,1,f4379107-9dab-4e25-a94c-142883aaecf4,1,2,1,3,ready,1,4,4
reordered_and_duplicate_notifications,2,6684ded3-8c62-4276-be37-1d49b16b478b,1,1,1,1,ready,1,4,4
stale_v1_after_v2_repeat_1,v1,9a0f2da4-365c-448f-8fe9-37005f8e007e,1,1,1,2,obsolete,0,0,4
stale_v1_after_v2_repeat_1,v2,32870878-9dcd-4d9a-9d14-f767d58ab364,2,1,1,1,ready,1,4,4
stale_v1_after_v2_repeat_2,v1,c7374ae2-bc84-4864-9aba-83ed638367c4,1,1,1,2,obsolete,0,0,4
stale_v1_after_v2_repeat_2,v2,206cd837-5a4b-4e45-be9c-facb46b1c85b,2,1,1,1,ready,1,4,4
1 case role job_id version attempt cycle fence state publications variant_rows physical_keys
2 commit_before_publish_and_worker_stopped effect cc13ff33-a32a-405f-9f9e-9586978137a1 1 1 1 1 ready 1 4 4
3 confirmed_broker_entry_survives_sigkill effect 0f014c3c-ab27-4f88-b195-043f23718200 1 1 1 1 ready 1 4 4
4 relay_crash_after_xadd_before_pg_marker effect c61da309-5d2c-4beb-958e-a2ca62f9939d 1 1 1 1 ready 1 4 4
5 before_effect_sigkill_repeat_1 effect 5f3097f7-410e-46ee-a05d-ee44f82b79dc 1 2 1 3 ready 1 4 4
6 after_effect_sigkill_repeat_1 effect 27d8895e-aa28-430d-98c4-6b2344140fd1 1 2 1 3 ready 1 4 4
7 after_commit_sigkill_repeat_1 effect 069ea9cb-342f-4339-80d7-4cfb2c5b0e21 1 1 1 1 ready 1 4 4
8 before_effect_sigkill_repeat_2 effect 11f50310-4da8-4404-9ac5-86010952cd92 1 2 1 3 ready 1 4 4
9 after_effect_sigkill_repeat_2 effect 3f39caa5-e08f-4716-92c2-ca31eea0004b 1 2 1 3 ready 1 4 4
10 after_commit_sigkill_repeat_2 effect 47446e4a-c8cb-41e3-b5e0-0da62d3c699c 1 1 1 1 ready 1 4 4
11 transient_retry_with_jitter effect 31bed0a2-2169-4b6d-8b44-ec63a1cfb448 1 2 1 3 ready 1 4 4
12 hung_child_killed_and_retried effect 8ee47c13-e328-43b4-aae8-f5f64d8d96dc 1 2 1 3 ready 1 4 4
13 permanent_failed_registry_and_audited_replay effect 9d7ed4df-7397-450f-ba54-4f1848525a8f 1 1 2 4 ready 1 4 4
14 finite_retry_exhaustion_then_explicit_replay effect 3a3d7792-9772-4a40-8eff-9db1515f60ce 1 1 2 12 ready 1 4 4
15 reordered_and_duplicate_notifications 1 f4379107-9dab-4e25-a94c-142883aaecf4 1 2 1 3 ready 1 4 4
16 reordered_and_duplicate_notifications 2 6684ded3-8c62-4276-be37-1d49b16b478b 1 1 1 1 ready 1 4 4
17 stale_v1_after_v2_repeat_1 v1 9a0f2da4-365c-448f-8fe9-37005f8e007e 1 1 1 2 obsolete 0 0 4
18 stale_v1_after_v2_repeat_1 v2 32870878-9dcd-4d9a-9d14-f767d58ab364 2 1 1 1 ready 1 4 4
19 stale_v1_after_v2_repeat_2 v1 c7374ae2-bc84-4864-9aba-83ed638367c4 1 1 1 2 obsolete 0 0 4
20 stale_v1_after_v2_repeat_2 v2 206cd837-5a4b-4e45-be9c-facb46b1c85b 2 1 1 1 ready 1 4 4
Binary file not shown.

After

Width:  |  Height:  |  Size: 69 KiB

@@ -0,0 +1,22 @@
{
"source_commit": "231d965b1a10a26a845f66a9c148a5ce7ff07701",
"cases": 19,
"observed_ready_effects": 17,
"observed_obsolete_effects": 2,
"all_ready_have_one_publication_and_four_variants": true,
"all_obsolete_have_zero_publications": true,
"successful_child_samples": 15,
"child_elapsed_range_seconds": [
0.2472045419999631,
3.999715418998676
],
"child_cpu_range_seconds": [
0.21908499500000023,
0.404055815
],
"child_peak_rss_range_mib": [
89.3203125,
91.20703125
],
"scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned"
}
File diff suppressed because it is too large Load Diff
+13
View File
@@ -0,0 +1,13 @@
phase,seconds,started_requests,statuses,successful_rps,success_p50_ms,success_p95_ms,success_p99_ms,critical_firing_names,warning_firing_names
warmup,300.238,1200,{'201': 1200},3.997,18.907,154.932,396.121,[],[]
api_brief,8.531,34,"{'503': 32, '-1': 2}",0.0,,,,[],[]
api_brief_recovery,120.002,480,"{'201': 479, '0': 1}",3.992,27.665,229.005,791.96,[],['ApiHighErrorRate']
api_sustained,185.636,743,{'503': 743},0.0,,,,"['ApiFastBurn', 'ApiSlowBurn']",['ApiHighErrorRate']
api_recovery,156.766,627,{'201': 627},4.0,23.464,250.67,1807.065,"['ApiFastBurn', 'ApiSlowBurn']",['ApiHighErrorRate']
ready_brief,5.004,20,{'201': 20},3.997,18.589,41.825,59.7,[],[]
ready_brief_recovery,59.986,240,{'201': 240},4.001,29.034,194.65,396.574,[],[]
ready_sustained,152.489,610,{'201': 610},4.0,21.872,150.518,429.458,"['ReadyFastBurn', 'ReadySlowBurn']",['QueueLagGrowing']
ready_recovery,303.953,1216,"{'201': 1215, '0': 1}",3.997,31.974,256.132,866.055,"['ReadyFastBurn', 'ReadySlowBurn']",['QueueLagGrowing']
backend_unavailable,90.001,360,{'201': 360},4.0,21.961,123.103,221.753,[],[]
backend_recovery,59.999,240,{'201': 240},4.0,27.369,154.043,469.609,[],[]
steady_after_recovery,268.806,1075,"{'201': 1071, '503': 4}",3.984,27.293,514.282,1556.851,[],[]
1 phase seconds started_requests statuses successful_rps success_p50_ms success_p95_ms success_p99_ms critical_firing_names warning_firing_names
2 warmup 300.238 1200 {'201': 1200} 3.997 18.907 154.932 396.121 [] []
3 api_brief 8.531 34 {'503': 32, '-1': 2} 0.0 [] []
4 api_brief_recovery 120.002 480 {'201': 479, '0': 1} 3.992 27.665 229.005 791.96 [] ['ApiHighErrorRate']
5 api_sustained 185.636 743 {'503': 743} 0.0 ['ApiFastBurn', 'ApiSlowBurn'] ['ApiHighErrorRate']
6 api_recovery 156.766 627 {'201': 627} 4.0 23.464 250.67 1807.065 ['ApiFastBurn', 'ApiSlowBurn'] ['ApiHighErrorRate']
7 ready_brief 5.004 20 {'201': 20} 3.997 18.589 41.825 59.7 [] []
8 ready_brief_recovery 59.986 240 {'201': 240} 4.001 29.034 194.65 396.574 [] []
9 ready_sustained 152.489 610 {'201': 610} 4.0 21.872 150.518 429.458 ['ReadyFastBurn', 'ReadySlowBurn'] ['QueueLagGrowing']
10 ready_recovery 303.953 1216 {'201': 1215, '0': 1} 3.997 31.974 256.132 866.055 ['ReadyFastBurn', 'ReadySlowBurn'] ['QueueLagGrowing']
11 backend_unavailable 90.001 360 {'201': 360} 4.0 21.961 123.103 221.753 [] []
12 backend_recovery 59.999 240 {'201': 240} 4.0 27.369 154.043 469.609 [] []
13 steady_after_recovery 268.806 1075 {'201': 1071, '503': 4} 3.984 27.293 514.282 1556.851 [] []
Binary file not shown.

After

Width:  |  Height:  |  Size: 187 KiB

+404
View File
@@ -0,0 +1,404 @@
{
"source_commit": "7282aa8b18cc33cf243cc8d82ff2fdd71079766a",
"runtime_versions": [
"752a5e6c0d33c2856f7ac8e66074e4adfb9a699d"
],
"elapsed_seconds": 1823.238,
"sample_count": 172,
"sample_gap_seconds": {
"min": 5.110785245895386,
"median": 9.663292169570923,
"max": 38.223085165023804
},
"client_quantile_method": "nearest rank of successful end-to-end writes; not histogram interpolation",
"phase_assignment": "request start timestamp inside measured hold; orchestration gaps excluded",
"phases": [
{
"phase": "warmup",
"seconds": 300.238,
"started_requests": 1200,
"statuses": {
"201": 1200
},
"successful_rps": 3.997,
"success_p50_ms": 18.907,
"success_p95_ms": 154.932,
"success_p99_ms": 396.121,
"critical_firing_names": [],
"warning_firing_names": []
},
{
"phase": "api_brief",
"seconds": 8.531,
"started_requests": 34,
"statuses": {
"503": 32,
"-1": 2
},
"successful_rps": 0.0,
"success_p50_ms": null,
"success_p95_ms": null,
"success_p99_ms": null,
"critical_firing_names": [],
"warning_firing_names": []
},
{
"phase": "api_brief_recovery",
"seconds": 120.002,
"started_requests": 480,
"statuses": {
"201": 479,
"0": 1
},
"successful_rps": 3.992,
"success_p50_ms": 27.665,
"success_p95_ms": 229.005,
"success_p99_ms": 791.96,
"critical_firing_names": [],
"warning_firing_names": [
"ApiHighErrorRate"
]
},
{
"phase": "api_sustained",
"seconds": 185.636,
"started_requests": 743,
"statuses": {
"503": 743
},
"successful_rps": 0.0,
"success_p50_ms": null,
"success_p95_ms": null,
"success_p99_ms": null,
"critical_firing_names": [
"ApiFastBurn",
"ApiSlowBurn"
],
"warning_firing_names": [
"ApiHighErrorRate"
]
},
{
"phase": "api_recovery",
"seconds": 156.766,
"started_requests": 627,
"statuses": {
"201": 627
},
"successful_rps": 4.0,
"success_p50_ms": 23.464,
"success_p95_ms": 250.67,
"success_p99_ms": 1807.065,
"critical_firing_names": [
"ApiFastBurn",
"ApiSlowBurn"
],
"warning_firing_names": [
"ApiHighErrorRate"
]
},
{
"phase": "ready_brief",
"seconds": 5.004,
"started_requests": 20,
"statuses": {
"201": 20
},
"successful_rps": 3.997,
"success_p50_ms": 18.589,
"success_p95_ms": 41.825,
"success_p99_ms": 59.7,
"critical_firing_names": [],
"warning_firing_names": []
},
{
"phase": "ready_brief_recovery",
"seconds": 59.986,
"started_requests": 240,
"statuses": {
"201": 240
},
"successful_rps": 4.001,
"success_p50_ms": 29.034,
"success_p95_ms": 194.65,
"success_p99_ms": 396.574,
"critical_firing_names": [],
"warning_firing_names": []
},
{
"phase": "ready_sustained",
"seconds": 152.489,
"started_requests": 610,
"statuses": {
"201": 610
},
"successful_rps": 4.0,
"success_p50_ms": 21.872,
"success_p95_ms": 150.518,
"success_p99_ms": 429.458,
"critical_firing_names": [
"ReadyFastBurn",
"ReadySlowBurn"
],
"warning_firing_names": [
"QueueLagGrowing"
]
},
{
"phase": "ready_recovery",
"seconds": 303.953,
"started_requests": 1216,
"statuses": {
"201": 1215,
"0": 1
},
"successful_rps": 3.997,
"success_p50_ms": 31.974,
"success_p95_ms": 256.132,
"success_p99_ms": 866.055,
"critical_firing_names": [
"ReadyFastBurn",
"ReadySlowBurn"
],
"warning_firing_names": [
"QueueLagGrowing"
]
},
{
"phase": "backend_unavailable",
"seconds": 90.001,
"started_requests": 360,
"statuses": {
"201": 360
},
"successful_rps": 4.0,
"success_p50_ms": 21.961,
"success_p95_ms": 123.103,
"success_p99_ms": 221.753,
"critical_firing_names": [],
"warning_firing_names": []
},
{
"phase": "backend_recovery",
"seconds": 59.999,
"started_requests": 240,
"statuses": {
"201": 240
},
"successful_rps": 4.0,
"success_p50_ms": 27.369,
"success_p95_ms": 154.043,
"success_p99_ms": 469.609,
"critical_firing_names": [],
"warning_firing_names": []
},
{
"phase": "steady_after_recovery",
"seconds": 268.806,
"started_requests": 1075,
"statuses": {
"201": 1071,
"503": 4
},
"successful_rps": 3.984,
"success_p50_ms": 27.293,
"success_p95_ms": 514.282,
"success_p99_ms": 1556.851,
"critical_firing_names": [],
"warning_firing_names": []
}
],
"incidents": [
{
"signal": "api",
"action_started": 1795185997.2563121,
"action_finished": 1795185997.581769,
"restore_started": 1795186183.218286,
"restore_finished": 1795186185.962821,
"timings": {
"ApiFastBurn": {
"detect_seconds": 73.368,
"resolve_seconds": 39.261
},
"ApiHighErrorRate": {
"detect_seconds": 44.859,
"resolve_seconds": 127.152
},
"ApiSlowBurn": {
"detect_seconds": 84.651,
"resolve_seconds": 127.152
}
}
},
{
"signal": "ready",
"action_started": 1795186414.030848,
"action_finished": 1795186415.9932559,
"restore_started": 1795186568.482792,
"restore_finished": 1795186569.2213361,
"timings": {
"QueueLagGrowing": {
"detect_seconds": 47.281,
"resolve_seconds": 181.174
},
"ReadyFastBurn": {
"detect_seconds": 76.71,
"resolve_seconds": 235.563
},
"ReadySlowBurn": {
"detect_seconds": 85.981,
"resolve_seconds": 303.95
}
}
}
],
"firing_intervals_unix_seconds": {
"ApiFastBurn": [
[
1795186070.950197,
1795186225.223615
]
],
"ApiHighErrorRate": [
[
1795185908.024147,
1795185996.921485
],
[
1795186042.440471,
1795186313.1144419
]
],
"ApiSlowBurn": [
[
1795186082.232623,
1795186313.1144419
]
],
"QueueLagGrowing": [
[
1795186463.2738628,
1795186750.3954089
]
],
"ReadyFastBurn": [
[
1795186492.702774,
1795186804.783844
]
],
"ReadySlowBurn": [
[
1795186501.9744341,
1795186873.1713321
]
]
},
"short_spikes": [
{
"signal": "api",
"action_started": 1795185864.820739,
"paging_events": 0
},
{
"signal": "ready",
"action_started": 1795186346.6159208,
"paging_events": 0
}
],
"collector_rss_mib": {
"min": 238.91796875,
"max": 239.03515625
},
"collector_max_queued_packets_sum": 64.0,
"collector_queue_capacity_sum": 64.0,
"backend_outage": {
"started": 1795186883.262524,
"finished": 1795186974.814038,
"before": {
"at": 1795186883.262471,
"phase": "backend_before",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 0.582676,
"worker_busy": 1.0,
"worker_attempts": 92.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.14903846153846165,
"slo_good": 646.0,
"slo_bad": 225.0,
"collector_received_spans": 163009.0,
"collector_sent_spans": 160474.0,
"collector_received_logs": 15246.0,
"collector_sent_logs": 14928.0,
"collector_failed_spans": 402.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 250630144.0,
"app_memory": 452972544.0
},
"alerts": []
},
"after": {
"at": 1795187083.355521,
"phase": "backend_after",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 1.890634,
"worker_busy": 1.0,
"worker_attempts": 132.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.20124999999999976,
"slo_good": 684.0,
"slo_bad": 226.0,
"collector_received_spans": 170729.0,
"collector_sent_spans": 165537.0,
"collector_received_logs": 16224.0,
"collector_sent_logs": 15564.0,
"collector_failed_spans": 826.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 250642432.0,
"app_memory": 452603904.0
},
"alerts": []
},
"ready_count_before_backend_recovery": 890,
"successful_writes_during_outage": 360,
"restored_trace_id": "60fba0bcea6e4b9fa2aead15e786b16c",
"queue_bounded": true,
"collector_rss_below_limit": true,
"send_failure_counter_counts_attempts_not_unique_loss": true
},
"preservation": {
"acknowledged_messages": 6370,
"accessible_messages": 6370,
"accepted_uploads": 359,
"all_accepted_uploads_ready": true,
"sampled_originals_match": true
},
"final_30m_sli": {
"api_good_ratio": [
{
"metric": {},
"value": [
1795187374.124,
"0.8992513640400964"
]
}
],
"ready_good_ratio": [
{
"metric": {},
"value": [
1795187374.131,
"0.7983193277310924"
]
}
]
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+553
View File
@@ -0,0 +1,553 @@
timestamp,phase,status,seconds,trace_id,message_id
1795180069.9570692,warmup,201,0.16097378730773926,e776267093834c7b89d8d53fd3c6f8b5,2d1f4e3e-8b85-4177-a48b-657ac153f43c
1795180070.212939,warmup,201,0.043734073638916016,eb474a2a92eb43989227a6f21537ac21,8afc44f3-b628-47ca-a48d-c38434e6d664
1795180070.459362,warmup,201,0.2732198238372803,0ff25933db8c40428275ec65a3905bf8,b3d8b459-e5e8-4e85-be31-8ca4aa62b4f1
1795180070.712351,warmup,201,0.035295963287353516,cc60cc8a82014effa9f831f3ad2909d3,bf276116-3268-4256-a3ce-4f4a8b48608b
1795180070.957216,warmup,201,0.06459498405456543,6acdfc552b2242458bb20b805a12bfdc,e45fb1d6-eddb-4636-b765-d3f1b54ef47b
1795180071.211649,warmup,201,0.03757905960083008,96156e96fb4a462b919edffb002c1277,ebe8451e-7aa1-456c-87d4-d770b07e4baa
1795180071.460105,warmup,201,0.009978055953979492,be2e4d671bc646e3b0ee2b882d6dd4a5,0255c592-503c-4046-9624-35e44a618fc2
1795180071.717076,warmup,201,0.011640071868896484,c877e6f6a0dc4b0395715916ba76a283,fb9c0ad6-ae59-4cb5-9a50-df03f5fa5cc9
1795180071.9642081,warmup,201,0.020227909088134766,0fdaa448d3964c5e9a44469c6cb68277,4a96e606-549c-4427-baab-191bb0d36e5c
1795180072.21425,warmup,201,0.038519859313964844,43b80ef7af8044e9b2b1bc1234c15f13,8cd6c3e7-6451-473f-9259-1aca3d8ac834
1795180072.460448,warmup,201,0.04502081871032715,dbf31c750923429095b0196c95454728,56503f1b-d7c3-4849-8654-d53bb7af63f2
1795180072.7154849,warmup,201,0.031109094619750977,46f4da900e4a4e009656802df64f2c8f,91d88223-fdec-4c91-bca0-4d5b28d76604
1795180072.959334,warmup,201,0.02359318733215332,99017fc0c3804148954a692be6c93b41,5579e52a-c91a-46b1-ae3b-b0475bc94130
1795180073.208265,warmup,201,0.027540922164916992,316c362b4f4e4ccfbcd7ab3f211f8cfb,7139f79d-c498-4056-afc8-e5c1eaaaf9be
1795180073.45748,warmup,201,0.07488799095153809,feffea2172e7461cb4171f44ef34b361,a1f19e0c-0ff2-48c3-882f-dee7064313f5
1795180073.713435,warmup,201,0.015566110610961914,2547a0ea547441dbbe974d44b458241c,9324684d-fe9e-43a1-a84b-bae5523663b9
1795180073.9612842,warmup,201,0.07160663604736328,00228b48dcbd4a1c84e81aa4c977d016,0f6ea533-bd56-4ed5-8c78-dab41ee30320
1795180074.216365,warmup,201,0.032988786697387695,e807fef317064c769b2074bdb344bef4,7b5834e4-65e0-404a-b07b-a58346afb77d
1795180074.467131,warmup,201,0.02714705467224121,c8eab445c70342b78ab3399b605b83c7,c530b3a8-c5e5-4c69-a3d9-0f0a8e20eda7
1795180074.713984,warmup,201,0.026133060455322266,7e6544a5661349c29edf9af88d3c9fc7,3dd3ed27-4056-48ba-9e8e-21515de5d2a0
1795180074.967433,warmup,201,0.023201942443847656,9a9720dfbe2442cea02dccf4eea97599,dcf1e137-2670-4a59-a022-58bf88cd9356
1795180075.213536,warmup,201,0.027930021286010742,59642b7fe13d4635833fafa6207876f7,5ffb0a8f-0811-4fce-96ba-fa8e5414153c
1795180075.459161,warmup,201,0.01526188850402832,bdbf4b92112e4787890cf366c3f67e71,a03d1041-2060-4a43-8fcf-77da2645cbfa
1795180075.708655,warmup,201,0.12127065658569336,9e6b038f957743939b2ad4bdb3e7209f,3e944606-3b6f-4131-92ba-7b70a748f85a
1795180075.965733,warmup,201,0.05122995376586914,9234b276a32f47a29b9fce67868601df,2b4bf449-2982-4d13-85f5-0a549a65f716
1795180076.21755,warmup,201,0.033921003341674805,c0d749b56e5049888a86e50212011025,90afb79d-6102-46b8-98d2-b7ff3914345a
1795180076.460042,warmup,201,0.017657041549682617,2984009087fd4f39bb9bb7cf1fca8a46,d833b693-bb00-4cc0-8aba-c116773024e5
1795180076.708305,warmup,201,0.009041070938110352,138f1cfd9f8845098388b0ee14118da1,0c474577-7e64-46bd-b965-80e9d7cfe10b
1795180076.963747,warmup,201,0.012278079986572266,03b2408cb3e344a49ab3f04c9607f160,6435d985-2856-48de-b33b-e335146db38a
1795180077.208763,warmup,201,0.025561094284057617,f545f350e2064ab697c70fdf8c38f2f7,7d2ece38-e7a1-4afa-a4df-74f9e0804e58
1795180077.46699,warmup,201,0.018079042434692383,3f245fa447364122ba33abe0f2c09179,7047997c-e173-4736-aa5c-36394ebe762f
1795180077.71347,warmup,201,0.013484001159667969,2d3da404a33444689cda5369dff039c1,c5c4835c-4401-4f50-9554-664126393837
1795180077.9609568,warmup,201,0.011887311935424805,9d82a52c79664331aa13a40571badfcc,1faabd95-bbd4-4d1a-8d11-410ba137b9dc
1795180078.209839,warmup,201,0.012413978576660156,57ab3eaf40da43b780d29f19843cbfb2,65896890-75bb-4934-8bdf-99ca1f783b40
1795180078.466974,warmup,201,0.012367010116577148,cea396ac874b472b937087e684fad26c,d762933f-ef30-4869-9d23-f015363c6f3f
1795180078.7080152,warmup,201,0.01634359359741211,8574bb35d3d24cd1aec875b112e6cbf0,c6946cbf-85e0-4d06-927e-76f43486e875
1795180078.967406,warmup,201,0.017637014389038086,bfa65e35fcd04d238601a48a17b433e5,5cd65d42-15dc-48d8-b965-c0b70970e96c
1795180079.213855,warmup,201,0.02095198631286621,93c131a83a3049a8823296d254f8e444,772c2a2a-9d59-45fb-b235-3e487708fe9d
1795180079.466801,warmup,201,0.009394168853759766,3d7c6fb3c23e4a5c905c949efb549c05,fb4240c2-55be-41b0-a2f0-f0802bdc69ac
1795180079.713575,warmup,201,0.009850025177001953,6fc2e838823b4f56a5562957a968979f,2851c65a-763d-419a-ae13-74da0e76fc81
1795180079.9616141,warmup,201,0.09672403335571289,5677b6efb3034fd594a53a94fdb6e4cd,77c8df03-15b3-43fc-9a1d-386f9ff3e1c7
1795180080.210528,warmup,201,0.25554704666137695,c64242d9feee4b45af94b4a54297b725,b14682f0-d03a-4e5d-8e19-18f5544ef9b5
1795180080.459133,warmup,201,0.04615902900695801,5722b7e2ddc84215b9dfc1dfc63fefcd,b241e81e-ac05-42d6-b38f-a1f034400491
1795180080.710989,warmup,201,0.036536216735839844,86b110dcacc7437f8f63805c9a881dc8,c6a1ad13-e162-4bcb-a779-8a819232e4d3
1795180080.95889,warmup,201,0.02017998695373535,ce58cb069c9b4455979a226de5bb77d4,14b20808-646a-48ee-a811-38d17f73eeba
1795180081.216896,warmup,201,0.012832880020141602,893f8fd075df4a24adbaeb3424dd842b,b5b81a9e-9512-4427-8bd0-3f0fa5c7f6b6
1795180081.4668531,warmup,201,0.05256390571594238,2b3598c4da6041ab812aa3fad1f8878f,c065a623-dc6a-4f4d-9614-eb13f97966bf
1795180081.7096372,warmup,201,0.014587879180908203,8546f1a24bd74842b29ee4600bad0e2e,2b68798a-8710-4d8b-a511-995c37de2613
1795180081.9634979,warmup,201,0.017155170440673828,925a99f02c5044c49e5c914e565b07f8,44b041e0-814c-4f00-8640-3c398c8b601c
1795180082.2102349,warmup,201,0.023682117462158203,165c55115b8649c380b7552bae3f579d,9b01fd09-ed33-461a-8595-911e17590996
1795180082.466957,warmup,201,0.023247718811035156,a6ca3d77dab14658b899fef8fc4f984c,7e017c78-4c50-44d6-a222-a47a48a43fa2
1795180082.712265,warmup,201,0.015156984329223633,913d65477bbf43e4810325d8f955aa95,42cc1f17-29f7-4e4c-b5b3-bba9a9fc2c49
1795180082.966871,warmup,201,0.014148950576782227,b2278b9f6d014d9daa209d4de38cf1b6,af0c15dd-80d2-499b-a0a3-1983237d6379
1795180083.211093,warmup,201,0.022732019424438477,c649ef237f904d6cabbad2aa6672cdc7,deda125f-e316-4889-b197-3f099963cc39
1795180083.465457,warmup,201,0.014335155487060547,7236502f12884b019f03115817e3cfc5,9ac3c065-5c39-4844-937a-3dbe092990b0
1795180083.716855,warmup,201,0.014973878860473633,d5573690968947ddaa68bbd9dfc54fd2,1d6465fb-1b33-4057-a73d-f3a95cfd5666
1795180083.962472,warmup,201,0.013889074325561523,27c7826e18a6438db0e9c29cde09d003,ac4ffbb5-54de-4b46-8a8e-d1a5bbb24725
1795180084.216266,warmup,201,0.010313034057617188,3d133c89500445569c772981d7a6d8f2,dbc8e815-7ef5-44da-941c-a6cccdb42a1a
1795180084.466589,warmup,201,0.027796030044555664,ed047dff153e4088b1f929221a33eaf8,f74d0068-af09-4a33-8d8c-8008480b2bec
1795180084.716776,warmup,201,0.020221233367919922,8a65f7dc12ab49bbb8605bd6f602be4e,17a6f33a-9636-46e9-abe5-1221c741b91f
1795180084.959736,warmup,201,0.02201390266418457,609086211d9044ee91129ca6cfaa07fd,a534b4ee-e80f-441f-b2ef-768613aa654f
1795180085.208767,warmup,201,0.022257089614868164,ea4a411fb9c04271a5ad8e6b1f1168f9,e38ebaf2-6941-46a1-97d6-63d5202d4f11
1795180085.4657888,warmup,201,0.03608131408691406,dfee0690d5d749988c9bd39aa51a9ec3,1a6978d7-ba7d-4d7b-9b4c-6595bd07ab1d
1795180085.710897,warmup,201,0.010241031646728516,239f8a508e744cfdada3dd0492f45d77,1bce9275-d0a3-424c-aa80-f8d25483e609
1795180085.964003,warmup,201,0.015243053436279297,950f416c575744038378061499ba2b72,74641199-21e6-4de9-941a-37e489252e2c
1795180086.2077,warmup,201,0.021860122680664062,13da9df1a5814f4ca8da0cb28caeded9,c4ef5518-de9f-43b6-b05c-73a1f16f8cc9
1795180086.464135,warmup,201,0.04335808753967285,ffc1aa67328a4b648efee25dc0f80115,e3323242-ad8f-41ae-9942-bd64702c4fa2
1795180086.713995,warmup,201,0.013170957565307617,ff6993364f5545448fad2c42fb86f2e4,7b43da44-ea19-4ff7-972f-b48bd2b3ddea
1795180086.958281,warmup,201,0.01422882080078125,9def7c264a2749a3a0bad35a6cec362d,5d3391d7-c86b-4789-95b1-59414122b2b6
1795180087.211974,warmup,201,0.057511091232299805,abad7cc034f04605ae5e81a60de8014c,2bd49519-3a50-4495-bfdc-bf5dc787fd0b
1795180087.466767,warmup,201,0.010188817977905273,0c839d1587424440b7218b6581d0e536,ecc9ecbc-c3d1-4d07-9bdf-a4abb256aea5
1795180087.716816,warmup,201,0.03182816505432129,7c535a26c45341e4a586bd6ba39254e7,cfb59993-e9c4-44c2-b182-9b05ad6f15b1
1795180087.964291,warmup,201,0.014871835708618164,09482390f77944e6ab2c518c0e01bca5,b65376ba-6344-4203-98f4-4b828ede7d57
1795180088.2152479,warmup,201,0.014531135559082031,21374ca696c34dad8bd1743aa2acb6d0,2adecbe0-a793-4ac6-9ed7-d5d76b8665cb
1795180088.4627619,warmup,201,0.014763116836547852,3bc9b4a40c1b48d4b9ad9c7f78d1fe24,b87f059a-7ddd-4471-957a-ec8ffc1d037d
1795180088.709258,warmup,201,0.02199387550354004,b71531aab7b846b784ce55529938682e,825dca60-b7eb-41c1-b966-3708fd3c0519
1795180088.957341,warmup,201,0.012842893600463867,522ca628e4364233b9125e64327bc83b,a0d28109-e2d4-4911-a0b2-d44a6e8a5ce0
1795180089.208671,warmup,201,0.009891986846923828,8020a9c2e82b4342a5864ea2da0dbe2b,740898e9-2cb0-4add-b566-85c1b91a27cd
1795180089.466711,warmup,201,0.0107269287109375,15cfdf5c30ad4ddf91b2335a7edf488a,0769beef-2b9d-47b6-9411-8398fd812edb
1795180089.7104251,warmup,201,0.020959854125976562,7948119f04dd486aabfa051f99212538,fd8234fc-0b33-4ab8-93e0-2d81297293f4
1795180089.963606,warmup,201,0.012027740478515625,742a4471437c49d18389b06e805a4ab3,b94f2545-4ab8-4adc-a3f0-3be6d2651cb0
1795180090.209379,warmup,201,0.0351109504699707,485f834821ee4901800a56167fc57a58,e092a5ac-578c-4e14-953e-e1a0f35307d0
1795180090.466652,warmup,201,0.011600017547607422,530e48d202294f02a057575c96939e09,a4d65f08-4d6e-433e-a26f-829b382a7af7
1795180090.713542,warmup,201,0.02402806282043457,eb8528092fe340249c57b787262eaa7b,cf3c1f85-16f0-46b5-8967-ffb1ed355a34
1795180090.958018,warmup,201,0.031564950942993164,066321c9365c4ac389b0ccc0246b74ac,ada76196-4fab-4482-83fa-b1f3b5eecb9c
1795180091.20841,warmup,201,0.047250986099243164,6e9c7ee6a8224292a5309c9efcfc6fdd,c010493b-68c9-4411-a131-28f51549c637
1795180091.4665458,warmup,201,0.01369023323059082,4f35ff3427584bc599fffb38720c8844,a5a00939-38ad-4653-8c3b-eb2f6b6dbc4c
1795180091.712262,warmup,201,0.04745006561279297,9a10a79bd165470f9d80c1b1af2033e6,c24bfdf3-5d25-468e-9624-70f7dbcde2e6
1795180091.956745,warmup,201,0.014162063598632812,c720e30747a0488fa74b22944a2cf758,15343548-7567-4d6d-973d-5915cd560bd6
1795180092.2134972,warmup,201,0.041545867919921875,58a18bbe2e4c4fa3bbf977564dd3b070,c744740b-c846-461f-b148-dd573f0400df
1795180092.4640179,warmup,201,0.1579892635345459,5ceb3ba17b6f4b6fa82e83dd2b45a550,55f782d9-ba6e-4fb7-a0b1-2ef0b321a085
1795180092.7122822,warmup,201,0.018977642059326172,8cfe624196ad4a108d25effffb13bb76,37c97b83-8484-4fcb-9c0e-6cfab9469183
1795180092.957189,warmup,201,0.07108902931213379,17f94c7584e84f8998ead6d44f40e85b,9edc0874-26af-41e4-85a6-83f49cb22fc7
1795180093.2155,warmup,201,0.015893936157226562,4f50a2f75a004d2890d0444434b2c654,02165777-9f1c-4285-987f-eb71c5a1c4da
1795180093.466285,warmup,201,0.010670900344848633,1e70aae6212e4411bc77affcf76c9122,7ce7a83c-43e0-479e-920e-6c45ddd097c7
1795180093.716158,warmup,201,0.01917099952697754,e7f7d36ef7e34852a270e54064609d93,4edee8eb-99b6-48c0-96a5-f24d939999df
1795180093.9611259,warmup,201,0.033547163009643555,aa606a9422d84863bf1e7cce2e1e99aa,ba8d2d08-9473-490f-9fa9-569c37363808
1795180094.21668,warmup,201,0.017546892166137695,88519bbc68b745baafe3f587e050ed12,34cc6507-b740-47e1-a9f8-12fb125c7168
1795180094.463606,warmup,201,0.02402782440185547,f2cb2077364c42eb99f97cc6e9cc411d,c84a275e-a365-404a-88b7-7619802c0867
1795180094.7084239,warmup,201,0.06055402755737305,ae3a8b7725d94141a71f3a4bb7b6d3a6,b51d198f-c258-4c31-9fb8-699f4dd6a649
1795180094.963558,warmup,201,0.011219024658203125,cb338827715d447281ca1cfb7baa6192,b9eeefe0-18b8-4d41-9a56-7e1a264599ca
1795180095.2135942,warmup,201,0.017534971237182617,048f292244af4d128526d24968d99cce,abe30323-cc5e-40f3-b0c5-d2fd04779543
1795180095.464899,warmup,201,0.02784109115600586,ca74025698904eb7a7c1b881dce81fd9,cc146ef6-43f7-46f5-b6b7-ad8f33422a17
1795180095.712756,warmup,201,0.06147313117980957,3b9a5cc7e14846f190112798ee495f8d,582f8ca7-ed3e-4635-bd14-3f086ae088e1
1795180095.961024,warmup,201,0.02721571922302246,3f9e74f4846747249dadb045a84612f1,b9164fdd-cfdb-40e8-9972-a7bb15331b96
1795180096.215352,warmup,201,0.011113882064819336,230ef86450f5409f90b0f3b72f13477b,426eb3eb-84ac-4871-ac6d-edfb3edfd9e7
1795180096.465764,warmup,201,0.010383844375610352,92977e8d8ca94f42b9f82b6760f37357,379c5abf-56ee-43d5-b7c6-9e32af7b3f83
1795180096.713002,warmup,201,0.010370016098022461,41ead6618f494679b7b1dea567965717,481f589f-948c-4447-99cc-fde77608cb8d
1795180096.9571269,warmup,201,0.013314008712768555,48a1a94c03674a668fe50472b6ca524c,0e64fda1-874a-49e1-9c0f-06a137d3a3fc
1795180097.207125,warmup,201,0.03571009635925293,0855de458df949c0960a19a3d6f81481,3c40f46b-c28a-4b51-9f7a-0e3747104429
1795180097.4657059,warmup,201,0.025941133499145508,fc09a297445c4255b337f8581fb73d0f,3b06592d-e56d-4006-96dc-13fcce9c40b7
1795180097.711473,warmup,201,0.019830942153930664,cf215ce3ae6841678c323fafc036c13f,9f445733-c50b-4690-a50c-900e3804ef95
1795180097.956442,warmup,201,0.011531829833984375,abba07cb6ab64b16a586f2193a8646c1,b931ebd0-fb63-4562-9417-7ac764ca1dea
1795180098.2144608,warmup,201,0.014567136764526367,e8d2e2539d774d88aaa2c4473977c67e,df144bb7-3ed3-4e30-b65a-194064b53183
1795180098.4572449,warmup,201,0.08848404884338379,6bd7fadb57c9417fb94e1a3419ab8cc4,271ce552-30fc-4efd-8cca-4eba7419763f
1795180098.708143,warmup,201,0.0271909236907959,5afd4658539842928dacb103fcf9de04,cce0b416-63c7-4dd9-9493-f8fe4a860a10
1795180098.96469,warmup,201,0.08780908584594727,b62003bace334091a3b8aa60404be850,bba8ff6a-4a76-40e6-bd6a-281ec6a96868
1795180099.2078679,warmup,201,0.02692413330078125,9e650dbe0af2454eaa8998c57d0cf42b,481bd894-496b-4266-b6b2-0e44cc1426a3
1795180099.4654238,warmup,201,0.012391328811645508,8b69c1152c28444b8d903d040f63d724,d66a387c-8a71-484d-878c-c6039df6c7cd
1795180099.716196,warmup,201,0.013508796691894531,08a4e30d9cd040289e88f1f32e6a6cab,1b1ecdbe-9293-41d7-842a-873e3ae2c49d
1795180099.965012,warmup,201,0.03931689262390137,f81e66c40e4c40c0a3574da6f042285f,9c2c656f-1486-41ce-b120-0a55526c0d95
1795180100.2069771,warmup,201,0.027806758880615234,29c3c90fad8a4ed781afec70fc431013,018459b3-7916-44b2-82c0-c7390a0b7ac1
1795180100.460327,warmup,201,0.019770145416259766,a2d50f5e1f124fab88c3ff5019c37652,89eea49a-a704-4a3a-af3d-21309c592214
1795180100.713459,warmup,201,0.022834062576293945,dc517a51ea684bc993695fce6571b5aa,87ace08c-2856-4ef8-8482-1c81e698894d
1795180100.9633029,warmup,201,0.011710166931152344,ef190508073f479a9d57bdb4c9b9c1e9,64bec410-474f-4412-b34f-8aa8d7b9e3b4
1795180101.216318,warmup,201,0.01240229606628418,7ee886cbc5094be3881d8882c3dfe9a5,f5fa9bfe-6f92-4b28-a439-7937370a2ed5
1795180101.460207,warmup,201,0.011929035186767578,e1a6eb07742d48729e6d06a3cef34df7,3530d4ee-e604-4b01-8f12-7b1441f2d50b
1795180101.716116,warmup,201,0.010046958923339844,0c100de439214d3ca293b8917cf9c575,f725ce7f-93f2-4e97-be37-d41ef73982a9
1795180101.956589,warmup,201,0.01887202262878418,bd9178c605734da7bf53e692c7fa269b,a085ee86-d766-4b74-8568-5d3e1d9ceec3
1795180102.2150679,warmup,201,0.04256629943847656,7ab9325f6b5c4364ab96c9369224cf1c,6835f67a-7cd0-4d11-ab98-e40d44693689
1795180102.462175,warmup,201,0.01769113540649414,8f392006bf984a59aedef30ee800156e,26aa8e16-eba7-4696-b32b-8dfb3cf13d74
1795180102.709732,warmup,201,0.04403996467590332,344073f83eb3454faf345c87ae4c89f6,333d76eb-d994-45bb-a987-c2c146bb8562
1795180102.9586382,warmup,201,0.015874624252319336,ce2c60ba55fa4829a3a477511b333ffa,fd203964-ff78-425b-8c1e-881e0323ec05
1795180103.206547,warmup,201,0.013957977294921875,a577df3378ff4b5fae4c2e0f7bb8bd24,929b0cb8-2692-4995-8dcb-5c61cad260c3
1795180103.465699,warmup,201,0.06361103057861328,b1716a8aaa4d474eab1ea16c7ef710c9,f076ab04-d83b-49be-b026-73bf0c205d02
1795180103.7078888,warmup,201,0.011960029602050781,5bf8e68832564e3d8b0815af5d9f4326,6928bfbd-e29a-441f-a048-635a466c73f1
1795180103.95976,warmup,201,0.014333009719848633,d8bdd91a857745d6abe1ed3e19aea098,5e559263-dd52-4050-a13a-ae1441a79556
1795180104.210141,warmup,201,0.0119171142578125,ec5ed7bd28ba4cec925e85e563b22928,3170e5ec-95ff-4f0d-b8af-9b8803663b1b
1795180104.462445,warmup,201,0.013860940933227539,1cceb6665907453fa5bb35ae531e06ae,099f91a1-f991-4315-9816-1b8dac80fd90
1795180104.716085,warmup,201,0.027819156646728516,b569df314b2d4c28bdca00cc5638ec82,457f96f9-fe3d-4086-b0c8-bc1148502545
1795180104.9571462,warmup,201,0.010566949844360352,ee53243220954c65be8a449284fd22e8,148c93de-1a65-48b6-84f0-ae15b9be6d50
1795180105.2108119,warmup,201,0.01622605323791504,a8009e01d96f48dbac64193fd0fe557c,200a85ae-6619-4391-8f21-56ce2877b3da
1795180105.465821,warmup,201,0.01184701919555664,aa99ea688eab46af83e302c28ce7d756,2ab663cf-6898-4cc4-9e43-8e40ddf480f8
1795180105.7093928,warmup,201,0.051905155181884766,0aaee39c361546b3849fd152947d9cae,8272cf06-6e91-46a8-a1d8-9d08e65c331b
1795180105.9574258,warmup,201,0.016104936599731445,e862f37799a8410686508c3ed816d7ba,9de19f12-24b5-448d-949f-ad8adbc32650
1795180106.215368,warmup,201,0.007997751235961914,558cd74d71da4cfeb1e6785e4678b33e,b8b88a48-1e5f-4f27-b46b-edf215c0e51f
1795180106.465971,warmup,201,0.016330957412719727,8ca7061c259749d890c2ae7a24ee10ea,a0e5fd6c-1db4-4898-987c-16f37224951d
1795180106.7134972,warmup,201,0.01929187774658203,96db26073ac04a9c8483d94d36d648c5,ba3b1aaf-9966-443c-8bad-8f3f5cbe772a
1795180106.965224,warmup,201,0.016424894332885742,49c438ff3cc34374a64df5a646ca0064,c04e7e61-6934-4b15-9f27-6992af1e740c
1795180107.2087429,warmup,201,0.13824105262756348,91a3a6e513ef48e091e5fec6cadc2651,c15ae37c-a0ca-4002-94ab-63c613b8a3f5
1795180107.465683,warmup,201,0.06748700141906738,a0b0fadee95b42d9846c6bc19f6a3425,fe21da70-979c-4f25-bd7b-ecec462a4f6c
1795180107.716597,warmup,201,0.043663978576660156,d0a18a5e9d3844609e1846de52fdc334,09b03827-9758-4c19-96e9-02245b95408a
1795180107.9577112,warmup,201,0.05071878433227539,8bc17d2ac861457d906edf6bb733aaa6,d40b8a9d-314e-4e6f-a235-53179160e38c
1795180108.215978,warmup,201,0.016988039016723633,b9d90c9fef9b4cd2b40f0d01f960a800,c2cad002-8df9-440b-8d06-8e9edb144d2e
1795180108.456348,warmup,201,0.013724088668823242,dd7f64e94a8e49808c688bc6e7705f9c,49bf1186-56cd-4c5f-962f-c1cce4115bf9
1795180108.716038,warmup,201,0.014075040817260742,46027f62899f44aeb5b997155d9dacd8,7800d167-6a9d-4299-b585-a8e422b6ed15
1795180108.9656162,warmup,201,0.029653549194335938,c0328e86f791432aacbd1351a7778276,c10794fb-2fbe-4254-b6d1-68c662d03d3a
1795180109.2134242,warmup,201,0.011368989944458008,78d77602d48648b9bb473b838ba80d94,0ffa1fa7-ca44-40ef-906b-91d7e6adac7b
1795180109.466222,warmup,201,0.011855840682983398,5a4539f748ee44139ed31d0523512e4a,03ed8cf0-6572-4420-a46f-f31c353d5ca7
1795180109.711676,warmup,201,0.016754150390625,ada1f3a756354017b1b3606e73a5e568,530d5f4e-8a90-400c-bd80-49c3282c6409
1795180109.957644,warmup,201,0.011628150939941406,069abe6e4e40491499fb9c0e060b1f02,fd454afa-45a6-48a1-acbc-a30f86ea1611
1795180110.209734,warmup,201,0.01928114891052246,feac37a20259432c87e036d5a3736f05,6c0e6316-6d64-45ca-82c8-a45d07998b22
1795180110.4659588,warmup,201,0.022558212280273438,83931cf8c4304d13bbbcb09fcbd09a2c,fe5bbefc-b090-4206-940e-53fa504766b7
1795180110.7111838,warmup,201,0.026392221450805664,c86accc4673a4021b5846f199e507dc0,aaad07ac-8808-4d6a-96df-28b4fe06feba
1795180110.9606888,warmup,201,0.02469611167907715,f2617744f0374aca828bd5e16a522317,a020e190-4f8b-4ffe-9ba6-454b67dab903
1795180111.206548,warmup,201,0.025948047637939453,d08ad39890994932b23b2e165e0cab6a,26fa8160-8bf3-4577-be99-c151ae1f91f4
1795180111.464929,warmup,201,0.021355867385864258,4ff40371a3fb4209a7995e9993296026,6f20343f-07e3-4fb3-8413-55ca6218b45f
1795180111.706388,warmup,201,0.011980056762695312,1ba5d021fa704d43a494e01106455fff,889d6919-d410-4fb9-8e6b-d0313916c9c4
1795180111.965568,warmup,201,0.021121978759765625,9026ed4c64eb4cfe9d358a97604851e7,4022af74-df48-47e2-a3a5-ce4c4eb29cd8
1795180112.2073479,warmup,201,0.016837120056152344,c02d63b9da53468cab2945d158c88217,43c2697b-fa58-47cd-8aa8-0017553f1a0f
1795180112.463627,warmup,201,0.009462833404541016,0b96e9c8026b4acfa6803e63f5d78270,61b8558d-3fbb-4fb2-a19e-143e173783b8
1795180112.713996,warmup,201,0.015658140182495117,bc6d171122fc4a82831317d5081c8c71,2564d01e-c563-4c1b-8d48-9f4c7625f03d
1795180112.959977,warmup,201,0.03197908401489258,26c142b3d1f346afa03037a9fd03f264,f21f69d9-3855-492b-8e12-c1c3173ff82e
1795180113.461932,warmup,201,0.3604569435119629,e0de43a93b9747ed8135916b9161edb8,88a841fe-a14d-442d-96ca-f9ccbbd90a85
1795180113.963574,warmup,201,0.0680840015411377,7f5ddf5140ca41eca9f99005e3be922a,1d915906-50a8-4e50-ad96-d955eae5bba0
1795180113.2087371,warmup,201,0.8404510021209717,266ef7a83b1f4ac6943f15629354bc1d,46a8f038-4ecf-43c8-af7b-a76681b0b7ac
1795180113.714091,warmup,201,0.3419630527496338,e3408be2e18243fe952dffc3a29d9f76,6bca83aa-886a-4e16-87ff-c4cbe343b04b
1795180114.21387,warmup,201,0.04325413703918457,531bfc2b57cb4a4abd2bace25b1b206a,56501d24-4338-4a51-ae9c-d571359a5a3b
1795180114.459767,warmup,201,0.07159805297851562,704037cba11f4f87954b914ca2dea4d7,9b7e9ffa-1018-4469-95ac-ae45816e1db3
1795180114.713313,warmup,201,0.013693809509277344,a8aea998a4a14b068536acb95950bd6f,ea6691a4-cd8b-4aba-9200-27596e23d8f2
1795180114.9657989,warmup,201,0.019134998321533203,f16be51009bf4267afffe901c6f9ec25,15bd426d-2b24-433f-91a9-b50b78a05afd
1795180115.210476,warmup,201,0.015871047973632812,8a59d5636bd849c195436327cf03d6bb,7a92fb63-0ea7-4b97-97e7-627217b57b48
1795180115.4618099,warmup,201,0.017672300338745117,8d0924d1e80648d99ad4b02f3743e748,ae81b2fc-f535-4164-9d0a-849757f3de7e
1795180115.7133892,warmup,201,0.23829078674316406,ee868fc316c44711899a2a7eac1f983f,d4d6bf84-798f-4719-8ce6-7b4f71e3edea
1795180115.963288,warmup,201,0.06793808937072754,be960e7d72fb41d686edb4c1519cd34e,a4a8dbb8-dd44-424b-8d44-94338319fc2e
1795180116.216624,warmup,201,0.011794090270996094,f1becd1850aa459188902bbfe85dc317,195060c9-3ab9-4947-aa5c-90fc6e708878
1795180116.465261,warmup,201,0.010736942291259766,ccfe2471b6094bdc98d8f7c7cc5215c9,8598b485-22cf-494f-8e7e-093045e91e0a
1795180116.7087119,warmup,201,0.019225120544433594,6490a5b10731467fb70b893217fbe7a5,fdde1515-647f-46ed-be62-61a32f012d97
1795180116.957767,warmup,201,0.012019872665405273,c0f622e8a99846f1897ce6d9e4d24109,2efc878d-0126-4ee5-99c4-b75f1b08209d
1795180117.2127738,warmup,201,0.023031234741210938,245477e2a0714376b674cd53912ff0cc,7396abb8-7d06-47be-bf35-caeb9104a6b1
1795180117.465743,warmup,201,0.03672194480895996,b699aef6a81a4583ace9caf63f684ad9,5b27c6e2-7533-423b-ad6a-6ffcef894efe
1795180117.7158148,warmup,201,0.01888728141784668,162f7b95191d40f59651888ee71f53a7,72779180-5849-4054-8d1c-e9f50e31dd53
1795180117.96484,warmup,201,0.04443693161010742,41e9e30f1cae4d4bad5e3e51fb7fe6e8,a9de2cb7-13f6-4d4e-8418-8394733c75ef
1795180118.2133079,warmup,201,0.023710966110229492,3698080dd9de4cb4a3b9087e6f736e52,00a5f651-45e7-4306-90e3-034890c0810b
1795180118.460449,warmup,201,0.029680967330932617,ff85e90c255c47d0a8648258d71bef3e,cfc32a1c-8eef-42bd-bbb5-7cf95dc1971e
1795180118.7111292,warmup,201,0.02736687660217285,cc2f0f325db64165b0e63db66ef2e912,946f661d-727d-420c-b2c1-10ccc5721ccb
1795180118.964362,warmup,201,0.014344930648803711,cd431bb740694c60a71b0c05f8af6974,0ffe6ffb-c402-4b6a-87c3-9bb8094ff16f
1795180119.211197,warmup,201,0.013164043426513672,d57df0e48f9848a0bcce0b113bd85726,4bf5b6f4-b8dc-4337-91e8-06d9dfb75af7
1795180119.4637692,warmup,201,0.011598825454711914,86002fec5caa4665b801fcf8dcb2e433,f863f2b4-5df2-4195-b2c8-9b153c392a53
1795180119.714405,warmup,201,0.024904966354370117,75aea0e8c93440a0b64f74afd2bab8e6,29929580-e108-4bf9-abd6-845383375440
1795180119.9656231,warmup,201,0.024814844131469727,f552643138364b199afa14b6418efd65,963ac84e-6238-44d6-90a7-959c7727efcf
1795180120.209861,warmup,201,0.019379138946533203,301dbfe0ea9742d8a6e972e148164e86,7b40e23c-22a1-4645-87d7-2cd7ecdab5e0
1795180120.460503,warmup,201,0.012269973754882812,56d19575ad6b4a159d29b2f28035a292,ce004c67-86e3-4611-b2d3-410c60eb29e1
1795180120.707459,warmup,201,0.016863107681274414,eb32c8837103450590381b57d9eb8a29,d4ac9a5b-353b-4cda-a097-06c588a077fe
1795180120.956764,warmup,201,0.03303384780883789,1a549a26298f4650b8ff2effb4231aa3,304dae29-1b9f-45c3-8f80-c3ee5ab8291f
1795180121.206913,warmup,201,0.01592087745666504,7d5af7fc0b024ea5825a6ed2a2638e50,5e90cd2f-4070-4810-a1a3-e27604dcf771
1795180121.456815,warmup,201,0.011278867721557617,76f880ca36c241d888ee2524fc8b2b3d,8d9e023a-99d1-43ca-ab5f-3dd6d9a5df32
1795180121.71327,warmup,201,0.025320053100585938,3df813a8d9c443ffbb44b524bbb916e8,28c64eb0-cc2e-4f71-8ede-8e852d15c2c2
1795180121.95684,warmup,201,0.00838017463684082,5531422685b943a6a369e608c058fc57,591f8ae2-343b-4273-a2f8-b6cf48646f32
1795180122.206609,warmup,201,0.00858306884765625,bb7288aba8434bba9fc41666e0904fc1,24f71b02-74a2-4901-8200-9cd545dcd2c3
1795180122.462899,warmup,201,0.015925884246826172,69d4f4e4b7544c409204aec3d83da9e7,e74f7431-65b2-4e35-853e-cec86896c78e
1795180122.709456,warmup,201,0.010297060012817383,7ae458c18ec5447c8ac6c6118bc18fcc,a090d8ab-702a-4b3f-a0ea-d4e383b45cd2
1795180122.964902,warmup,201,0.013642311096191406,a7a71c5177a6460a8c2b7a8a5127a8a5,bf2acb88-b590-419b-8578-3201c3381a73
1795180123.2154398,warmup,201,0.012851238250732422,b41e6dd0417040dfa5ef0bd7a51d4d5b,4486c43c-b469-4899-9d82-2d6afb8f03c3
1795180123.455821,warmup,201,0.013101816177368164,0e0788c63dd3458a988dc582fc97093e,0914e390-c13f-46b3-bcdf-998b9deac79b
1795180123.708153,warmup,201,0.020342111587524414,8e3ad121c9184193b84eec5c7700ff37,aa11cd4a-7649-47d7-ad4a-4daba45c64a9
1795180123.955742,warmup,201,0.05740213394165039,c1a8efda79ec44e0a3c108a65ecfe280,70604d3a-50e9-4c70-9337-04b65fdc4809
1795180124.208821,warmup,201,0.016861915588378906,cd2340f1dfd248ebb7d5356f79f8f810,561a47d7-b772-43a0-9033-1868fa5ac4aa
1795180124.459167,warmup,201,0.08424186706542969,ce5f4ebaa39a48cc9925177ddd46b951,60f54ad3-215d-43a1-8118-6c72a5efa4c3
1795180124.7120728,warmup,201,0.0500941276550293,12adf6cb9cfa46e4b59d0d827ac86be7,f523c477-67a0-40e4-822b-bd6cda069068
1795180124.955792,warmup,201,0.46666622161865234,cde8f775be3e45d09e4f031ba0836095,ac5ed17e-b55c-497d-a88a-93d5612ac03e
1795180125.2137432,warmup,201,0.21810579299926758,7789893b847749a39412cb64dad122d0,c7c84df7-5b9a-4f26-a31e-ebe6ca26d14e
1795180125.4653718,warmup,201,0.4004700183868408,4c692c0c992e4cb4983d702cb7f2906d,ab70beeb-6847-4079-a880-272308582b3d
1795180125.7116349,warmup,201,0.2136549949645996,4a952313b6c241e7b24648f925c91c87,396d5aa0-9d6d-4f66-b11c-47654ba5060a
1795180126.20828,warmup,201,0.022182941436767578,4b2f49632f6f4378a862a76bf2b0a11a,287778ad-cbcf-405c-9781-76c32785d4f8
1795180125.957431,warmup,201,0.4076690673828125,c19fbf5dc41f44678bf65a75718f4763,ead5c9c8-736b-4cca-a826-797020bc4952
1795180126.465112,warmup,201,0.11259102821350098,35523a266edf48bd943263384c9787ad,70c1baa4-3067-492c-97e8-b70ddfd400eb
1795180126.70634,warmup,201,0.1282970905303955,08db262afecd475faffb5c5151eaec0b,5a306129-f313-4586-afc8-6ab0d70302af
1795180126.956701,warmup,201,0.08037590980529785,8b66a1e2b3fe4bc5aa66698dc7534c95,58c38c8b-047c-43ee-8e2e-a105acc3960f
1795180127.213266,warmup,201,0.0364232063293457,023627bd9b8f45bc8335c9d02ee6a0e2,c209d4bf-212f-4267-b115-8456a3b6cd8f
1795180127.460497,warmup,201,0.018844127655029297,d66c88144827455095a11c5bb3a8ab75,74a961b6-8a03-4592-8477-8c79765f6135
1795180127.70784,warmup,201,0.013520002365112305,a5dec999bcac4e60ae5bc69113c4b97c,0694641f-063a-47ab-ba69-446a24b0302d
1795180127.9577322,warmup,201,0.014217853546142578,2968ad48558e4ed5a5a282176ce12a28,211849a6-68f0-4974-a009-8f3056aac6b0
1795180128.2072031,warmup,201,0.012610673904418945,8e65ac12d21243be946656474b9be0a9,e612a563-7d2f-439f-a64e-6ad7902b758b
1795180128.460282,warmup,201,0.023057937622070312,66f10fbc943e4496a3ff400a3e46ed0a,eedc2fc5-c6e8-4613-88a6-77a6637c4584
1795180128.713231,warmup,201,0.03037095069885254,31fd1195cb3847d08974df206fc6b0d7,c45a7609-ad92-437f-889c-ce7962d075d8
1795180128.956662,warmup,201,0.028537273406982422,9e68be6d44d64037bf3006185c2d5d23,4d731559-f9ce-40c0-95ef-8633fadcdf99
1795180129.211051,warmup,201,0.0380401611328125,03b2f65668d34f15b6c9546be47fec86,94b225fe-050b-4be4-b9bc-d85c7180abd1
1795180129.463252,warmup,201,0.03919506072998047,9d7008be8e4b4e6b84dda2204dd3a0f7,ca4fc962-d814-42f1-81ef-5ac07d99f4e7
1795180129.705261,warmup,201,0.01246190071105957,5b7a9e05ef18435caae5c26fd0fbba55,c780199c-588a-4089-8e1a-b9e29040dc94
1795180129.958072,warmup,201,0.03585696220397949,80f818ad28014402851cf0dc4fe6b6fd,70fe8691-9a92-40aa-aba3-d66c7eaeb224
1795180130.2058141,warmup,201,0.09956502914428711,29d9e9488c834b89b5811683ef978cdf,72350268-eb1d-48f7-9c64-20b973f464b5
1795180130.465339,warmup,201,0.025896072387695312,d7b5a37e9f774ba09b755dc347a6a543,2abf41cc-ed1d-42fb-bd0b-c4dd09ac95a1
1795180130.7066429,warmup,201,0.011258125305175781,138cf5cf556a46bd8c5996af9771a756,fcf61d52-05f2-40f8-b719-028a7d3799cb
1795180130.9631832,warmup,201,0.013113737106323242,6cd0c7e99e144c189130e48bd586ebc2,e384af86-879a-49eb-b88d-27fa49ae5387
1795180131.2116659,warmup,201,0.03482413291931152,4bcea84186e24a79a32ea26db27b9110,652c07d3-b5a2-4b5a-9739-34d2340bbedc
1795180131.4630752,warmup,201,0.01401972770690918,dd7c079671544c79848adf21e81287b3,04f4b354-d35e-44c6-9cdd-5c0701c50001
1795180131.707235,warmup,201,0.01443791389465332,7f24ba99e54449eaa6dd61923148169e,2620aa56-3e44-43fe-9f07-9608321b14d6
1795180131.9631941,warmup,201,0.010252714157104492,5bba090e4ec542649013171deb048bbf,5cb08e95-e5a4-4ab9-bd2e-57c5f7024413
1795180132.213757,warmup,201,0.01481318473815918,e7d645f2882348cb9e340b8993c8bcdc,b82f4159-b720-44f7-bf06-8d5d91c7733a
1795180132.463203,warmup,201,0.024817943572998047,352c5922524b47a386900aa4e676ee34,6c286b53-7960-4d83-9885-04ac3de75eb0
1795180132.7094731,warmup,201,0.008898019790649414,f4035257b55c41d5921cfecea8966176,574fb6a6-2aeb-465c-831b-a64ea30727c8
1795180132.956722,warmup,201,0.010103940963745117,afd60764b6934ebd8a725d78bd3f141b,7972d992-f8a3-45d5-9bb5-4314a6553f51
1795180133.206574,warmup,201,0.015628814697265625,1721768d2aa64664ad3e327db144300a,1edafeaf-e173-41b6-8c3c-83f18a8166c1
1795180133.456523,warmup,201,0.011080026626586914,f15339e1ec4d4a6ca8e331011f988549,f72cc398-eaf7-4d27-8b08-c301779f1dfa
1795180133.71329,warmup,201,0.013360977172851562,c96a9745a1b74d58b3dab2cd5645c4a4,4ac57986-d879-420a-b1e1-244e561e2452
1795180133.96325,warmup,201,0.0173952579498291,6f2a1136ff1042d58b8f6cd077189097,cc1000c0-ad46-4409-a37d-6ff7df3275f2
1795180134.2074082,warmup,201,0.013551712036132812,1c33f13207324f7391e00ac92f57a94e,d2e5b4fb-af9e-4279-8584-8bf2c0d76a29
1795180134.4623308,warmup,201,0.007900238037109375,34e1fdd2284a4dc388d43c3da2725df8,880018cf-c8b4-4395-b5f5-fa1c8fa65273
1795180134.7132852,warmup,201,0.011044740676879883,6403402a1ce84f2f8806a914dd8215b5,61f00d79-ccb5-494e-bd71-d862ffeef5d1
1795180134.9608378,warmup,201,0.02025604248046875,0894586e34c54ab697c597b7421d034f,3244ed3b-f15d-472b-9e3c-8d3e924f5ff6
1795180135.207734,warmup,201,0.011043071746826172,1b06517ab2a444179728fb8dbfca0827,03d2f9ae-3ebe-4dee-a81c-6b6c406f9d2e
1795180135.455019,warmup,201,0.013014078140258789,fd6c4f6cf4ef4e1db2ecda69d20735cf,89bd0345-ec51-46d8-a006-ff0a2d410b5e
1795180135.712738,warmup,201,0.10438084602355957,8f09b5249863497e94b4ac5dd86ba4a0,c23262a9-cc13-49bd-8f23-795dccc65e49
1795180135.9571142,warmup,201,0.018002748489379883,f075fb7c0d434b67a026427a792aeec0,351ccb0e-cc60-45bb-ab0c-8ec9d6340b0f
1795180136.205044,warmup,201,0.011342048645019531,fc8eb864844d49689747e262be617164,f250da35-d8a5-4b94-af66-183bb0c14d81
1795180136.463886,warmup,201,0.022532939910888672,1e2cf1399e5947a8a3d01c53e008b506,346b4b23-4597-4a11-8206-cd77d95da8cb
1795180136.7051678,warmup,201,0.0228271484375,1e97029cac1f49d3936d82698cf0f18d,98396eee-0cfe-4362-b38f-479c9d56121a
1795180136.95592,warmup,201,0.01760697364807129,fa15a0606f634b378faef6d7851b21ac,5c6bad4a-710f-4f13-b688-1c694c8ba23f
1795180137.2061172,warmup,201,0.012992620468139648,ec2ffdb4bad8408f8203f6c45fbd2bfa,722c2c2d-e8df-4960-ad16-ba0705f57818
1795180137.4616551,warmup,201,0.011270761489868164,e03220cc975a40b2866f11bec428e421,18f123db-6131-4d5c-bce5-8cb6857af849
1795180137.7097511,warmup,201,0.01858997344970703,e38529e88ca9466aad28b852ee566d05,d6fb2504-540c-4455-8431-c5a800f7536b
1795180137.956691,warmup,201,0.015127897262573242,7ae2111789d54a03992356796a309d08,1f023142-499c-431a-a878-710b377794c9
1795180138.2149088,warmup,201,0.020531177520751953,5775e215d1484f9ebbb21fd79036e3b9,5f8dc182-154f-4d6e-8720-6475c7823c52
1795180138.45882,warmup,201,0.015606880187988281,e0f4da4791f041d0a1b626e48732b09a,8349e524-f496-4d09-bc55-b25ab82ac628
1795180138.7100549,warmup,201,0.012995004653930664,c3461393466d4972b02dfb9c01957188,5bde5b66-1212-4260-a66c-5f7657e5f10c
1795180138.9580622,warmup,201,0.01614689826965332,57c5d3fbe0be4fc49cb0fdad8aadfa92,cdf0eecc-6a30-4541-b9f5-ddef849546fb
1795180139.2118132,warmup,201,0.013647794723510742,767e969ca9014b6c9996531ce559df61,88609695-abdd-4bad-96db-9fe814702590
1795180139.4632142,warmup,201,0.009091854095458984,f86e3591b7e64a71b5553f371b7f40c9,de93cccc-76d6-493a-ade7-3ea0a0d5fb71
1795180139.714897,warmup,201,0.015331029891967773,cddf319c99ef408c82bfb4d343ef2635,8fa78445-3d8c-45a7-96ae-01a941f020f0
1795180139.9627159,warmup,201,0.01434016227722168,3fcf4fb003844ec8bb54ea1f5f733efa,6b4ac399-f00b-4119-ab42-0af7df3f357a
1795180140.21356,warmup,201,0.049919843673706055,78da2033cd6746b3aacecb1a52a94bff,43f75c5a-7986-4c87-b2be-643701e8a30b
1795180140.463584,warmup,201,0.013402223587036133,e9561b1d921a4901a0d76129b05447a6,0bd74017-3e94-4f66-8019-b964128d33be
1795180140.713254,warmup,201,0.016618013381958008,c8478479d45f43b6abf82abeb4fa83ab,1f35207e-990e-4b2b-9b12-8b2ca300e495
1795180140.962937,warmup,201,0.019067764282226562,51f2dcc6fa61439d9b90a247c2f15144,1a4ee6e5-a8aa-4b57-ac2c-f6d7a8605e02
1795180141.214133,warmup,201,0.011148929595947266,acbb62edba674fdbaf5c001a80efa143,1bed9e98-06d0-44aa-99ef-7a0ba428d72b
1795180141.462243,warmup,201,0.017611026763916016,a80c5bdac3b848ac89ab8d1132f29295,81969480-34ca-46ce-8724-090271f2d3ea
1795180141.713085,warmup,201,0.02792811393737793,4a1c8d5e101e457aa1bb3532c62715ff,2a4755eb-58d0-4e0a-8f04-c7e085bb720a
1795180141.963087,warmup,201,0.01384878158569336,adc2b0e3598945b4b1b6831405aea2f2,4437fa0d-be96-41dc-b191-332f69d6aadf
1795180142.204875,warmup,201,0.01084589958190918,09891c3bb41d4df9836d56c0ad0e942f,bca6e934-63af-4bb9-956f-e53c7205efd4
1795180142.464761,warmup,201,0.010806083679199219,660ab3f0b0ff49debf10a44783abe0fc,b83355bd-9cb9-47cc-8a37-295e91755f76
1795180142.71471,warmup,201,0.00988006591796875,3ba7601657da46a588e8a6e9ef71cd8b,20c7f9a3-b284-4af0-a23b-c16479768a75
1795180142.959651,warmup,201,0.015625953674316406,eb71302e239744f2a759cd4f4902a058,1c3ae2d0-920a-45e2-8914-0e71924f5a05
1795180143.213386,warmup,201,0.009329795837402344,ca97cb82e52544f6a5c4400f7bb5f3f5,bea54320-8738-46fb-9f20-0ebc92a9d599
1795180143.463989,warmup,201,0.013462066650390625,9ac1e64e71e84d41963aceb41d3c9331,959cec4c-2063-4e65-ac56-29b3733739a5
1795180143.707256,warmup,201,0.0188601016998291,d18b958f46c247569415bfab8c827026,5acac01c-4e9c-41c0-8153-81cfb3f4879c
1795180143.961851,warmup,201,0.01336812973022461,2f1a398627144fbdaf3f2e478d33b28d,e3962383-7d5a-4a92-877f-249f3bfb0f41
1795180144.2063081,warmup,201,0.011801719665527344,39a64faf76494e99a03a888cc7e2e4c5,042bd8b2-5e39-448f-8fb4-0f5278a920c2
1795180144.463094,warmup,201,0.012636899948120117,db742f1e32d6433fbdeb0ddd3da29f97,f5fd8b85-e55c-4e8a-8870-1d645bb46a20
1795180144.7092211,warmup,201,0.023552656173706055,64f7b49a036248d1a70980d117a0264c,019eb5a9-db89-4439-b044-849894838ff4
1795180144.960939,warmup,201,0.01747727394104004,a2f1fee7c68847e5ba5d2cb6e950c8c9,6992a0cd-0f41-4045-99be-c229b8450d68
1795180145.206008,warmup,201,0.02232813835144043,e5be693bf9654a7993d12aa16398e524,8061e634-e325-4d3e-9c3f-79f13edb06b8
1795180145.464622,warmup,201,0.025964021682739258,89689b110ecd442f8d823fd5ad1ef546,d225696c-a1ab-4f82-b65b-0ab6f411b87c
1795180145.714587,warmup,201,0.013881921768188477,e5b573ab906d4d47bc283bad0c283bde,d0cd8d44-e388-44dc-9939-31448705398b
1795180145.956403,warmup,201,0.017344951629638672,1b12f10f44a54f87a2f82e80da2ca146,1d61c6a1-87c4-4622-9f7d-a1c5b97f3f1e
1795180146.205685,warmup,201,0.01906418800354004,be5614f0646e4c6c927704b32c216845,9ba1718c-e5de-4c71-a971-e1a06705f187
1795180146.46311,warmup,201,0.023369789123535156,070ea00c630b49c39cd773bd022a2440,abcec0a3-3c91-4598-838e-9e4b098828e4
1795180146.7096572,warmup,201,0.11615276336669922,134e4f4d39724ce2b453908df6ee3c32,6c9588b6-3e39-4335-ab45-76980ca22058
1795180146.962498,warmup,201,0.015411138534545898,bc00ee055e934794aedf46848b6d9d4b,c293e145-6d05-4dd7-82c6-2c08595ce25f
1795180147.213019,warmup,201,0.01767706871032715,211a0e09715649478212fd645fe9c512,1733a12e-bb1c-4f91-a5d5-fb7d8e5254ca
1795180147.461827,warmup,201,0.08181881904602051,776ab497422a45e5947ddb531dc82433,8703b6e6-3869-4195-b083-a9bea68a7d37
1795180147.704898,warmup,201,0.010509014129638672,7c0b031447754345987cecade08e1e66,68a41089-abbe-463b-a62c-86014f741453
1795180147.955076,warmup,201,0.013746023178100586,533292e1cfc5425c80060b0f062c7e74,04cbf080-c35e-4ed5-89a1-02a9696d51a3
1795180148.205858,warmup,201,0.025356054306030273,1b139f377f83459caccc028d4b2911c8,77c18782-d7e9-4c28-951e-cc9f4bac2a8b
1795180148.4552372,warmup,201,0.01289987564086914,b98cae53f29045d19a8f133e79c9dac9,a0036146-0213-4c84-b15b-d5d8779684b9
1795180148.706578,warmup,201,0.02797412872314453,904ee8f5c1d9496bb40cf6d3fa23c656,73824605-99c2-4c59-8b63-82dc61294f03
1795180148.963203,warmup,201,0.011016130447387695,5543a1f12ea64c4c9c45d2551f6f8fc5,8e3b04ff-3dfe-411f-894d-5986c58bc61f
1795180149.2050078,warmup,201,0.01116323471069336,72ac534ce025497f8d8d1528baf61292,bfdc61c8-accc-4673-a0b7-4523936a42a9
1795180149.461789,warmup,201,0.02168107032775879,f42dc751108b4816a1a2776541958326,15b9d117-ce4b-4523-ad0e-2efad6034a24
1795180149.710597,warmup,201,0.01814889907836914,d3247bcd8b974042a193c4749779bcc0,509ae033-f080-4c6c-9c22-e26b1ab8c4f9
1795180149.9608488,warmup,201,0.015266180038452148,947d80b1b7f24134a13749b0f9054e7c,51a092bb-8aad-4f54-923b-8aeba78e430f
1795180150.213044,warmup,201,0.030508041381835938,c2df4f426f204f9cb57c75398cfcd2d6,70a7a0d7-e40e-4f4c-ace2-b9244ddc88e7
1795180150.459798,warmup,201,0.015178918838500977,defd7634ee5749bb9cb56938fffb6354,19f848ec-e61a-49ca-a9ab-ee159c377d14
1795180150.714624,warmup,201,0.01801013946533203,3b9ebbbbbc004bbbb1173a1decbccb00,68500f45-3a93-4b17-a678-82f4250ebb0c
1795180150.964399,warmup,201,0.014775753021240234,d7fdec7d34c5496e96d25c4e8b6f51a2,921a3c63-3103-4882-8383-1e4950642aa4
1795180151.214231,warmup,201,0.015651941299438477,a919ca5a526b4cdd91873a89b3249d7f,c75eb90d-4cd7-4419-a1ff-88b6634c5666
1795180151.456893,warmup,201,0.01569509506225586,7e65e6891eb749aca2e5cca772c2be51,3528cb80-0103-4249-9063-702d10cbd573
1795180151.712977,warmup,201,0.011497974395751953,5f9b3149a64640b79cb47627cfef6355,2d8814ca-6d6d-483a-85ef-5959661861a0
1795180151.9644082,warmup,201,0.009419918060302734,c8f9dd041f14439b8e0a88f48c057989,cdf1a975-bc52-4014-b534-59148bf6c73a
1795180152.2131011,warmup,201,0.014716863632202148,68761e2b15b64610b364c2a964c6d5a8,3c1cd970-d8f6-4049-9a7e-a574670f1920
1795180152.4601622,warmup,201,0.06042981147766113,0ce081bdcc584db8804a460edb6657c1,c0d66583-f650-4120-8f62-3d2c7ae54dfc
1795180152.706217,warmup,201,0.020451068878173828,a664c5f673114604aa95b08aa1b7e972,893f49c8-3afa-4766-b58e-c54efa12286a
1795180152.9611251,warmup,201,0.025645732879638672,569d9881d33b4e0ebcdb3b55833bdce9,b25ab34a-e7a2-42e7-8674-de4ae680db50
1795180153.204326,warmup,201,0.023772239685058594,d15bd802b80a4cf0b5c5b66fe0247211,ba2185ce-cfb1-480b-909e-a9776122877b
1795180153.462528,warmup,201,0.04242205619812012,fb3345a7c61d4504b216f861da365b7e,8ffba94b-38cd-4380-a563-8f19403b00b8
1795180153.7144508,warmup,201,0.024564027786254883,41c45e295c094077b016ed6bae9a2022,a132391c-bbb2-46b2-b424-bd6a93139ef3
1795180153.961816,warmup,201,0.03986001014709473,170e1f3a76bb41fdba945e2e71f6696a,30fba817-b082-4d0e-8ef7-752c23ee4af8
1795180154.20493,warmup,201,0.03506898880004883,5bf03ad7556f43718c11e5aa04c57eef,a590d9ea-3b17-4f7e-bd1e-a634d5905445
1795180155.218189,warmup,201,0.7187919616699219,2d064afc92574e9dbabc65af8987a7d7,abff9a4d-707b-4626-96c5-698361062611
1795180154.4665148,warmup,201,1.4753811359405518,670d7efd7b35456ba9b976db307daa78,1a183e3a-1b15-4d3d-94e1-74c22a1d02fc
1795180155.711974,warmup,201,0.2379162311553955,5bb3401f86f246dd8ee36d2e60e31858,1d37a14b-850d-469d-8b2f-8cf48b9c1c24
1795180155.0022368,warmup,201,0.965947151184082,ae126ae644014ec5b1936e416ed2a435,e52afb5e-8df8-4ed8-912e-d83a28d52c9e
1795180154.709724,warmup,201,1.354740858078003,e77df7dd28f84c3d95a7cc15d5e98456,d8df3654-4ccd-4a1a-a4fe-d747ed6bb174
1795180155.473359,warmup,201,0.5913128852844238,fa2ad4a0d89f44c58b539416eb37b3b8,01cce25d-695e-455a-9b89-3277459c6471
1795180155.963171,warmup,201,0.10228514671325684,a90640776b8c48d38e0ec681f22ac39a,0301e0fa-5b43-4ec3-bde6-339af08a4e2a
1795180156.214087,warmup,201,0.018687963485717773,7f967f771f324b2da7abe59c4daef1dc,1beda72b-b70d-41ba-990c-e5a062595310
1795180156.462667,warmup,201,0.018568992614746094,737f8ea6e6d84bd3b51cffe92b59a69c,53dfeaeb-b8bf-44b0-b526-5b16bff819e9
1795180156.7076569,warmup,201,0.02150106430053711,759761e721b142b9a418803b6df9c027,a06894db-6c95-46d0-8e17-60b650716376
1795180156.963069,warmup,201,0.020072221755981445,73a497e1f8cb490fabd21547c58e94c4,b008b068-3ff8-4bcc-8baa-b2aa568e0628
1795180157.210342,warmup,201,0.02046990394592285,8779184ae9ba4e23a3fc2a99f6980866,a15fd300-6132-4d3e-b1de-7264c7edb3f6
1795180157.4589682,warmup,201,0.032640933990478516,60829f4073814d08a8aebc311277ff33,82e61037-01df-4764-ac06-21e048a1596d
1795180157.713058,warmup,201,0.04641890525817871,84530fe1b329459ba8c05fbecc725d96,c23fb1ac-67fd-48f5-924d-6fad526fc69c
1795180157.960186,warmup,201,0.0545649528503418,b6e31f7e82314adebd2c96abb12976f6,00b781e8-1a6e-4d1a-ae34-75d2a1990179
1795180158.2074811,warmup,201,0.024766921997070312,2985a09493b1491c97cae490772bfc8f,d02ce7b2-e0a1-439c-8a02-aa57ad954b0e
1795180158.4595351,warmup,201,0.1045236587524414,921a1935688f49e5ab1828cd40a3c375,fd12108f-603a-44d5-8d93-d29675978d04
1795180158.987233,warmup,201,0.26446008682250977,93913d95e58246299d872da8dedd7920,5aee7bc6-a089-44af-9753-a647806fa999
1795180158.708724,warmup,201,0.6227638721466064,7b8b3b12ecde4fbcbeb074dff21b663b,cdb7a2a2-2392-4bb6-8feb-18f6c37c22cf
1795180159.221353,warmup,201,0.20409274101257324,2a411aca603948868e5b73cda6fe91e5,83299336-71c3-4f85-afd5-9a7d632903be
1795180159.463039,warmup,201,0.09139418601989746,26b02c0f71f24e5c8ddc96b9d1ae25a5,c32941b6-a35e-4780-91ed-17cec0715c14
1795180159.706288,warmup,201,0.028244972229003906,619307cbe4bf417da6c149d3f3cb5b8c,a5fa4e6c-2878-4e01-82b2-678857cfdfa8
1795180159.956794,warmup,201,0.012042045593261719,bb655577487a4fc8ace4f902c8e8c199,5ede45e8-2c45-4a1b-a9bd-6f04fdb58c97
1795180160.213904,warmup,201,0.05432319641113281,8f7670e1d09241e8937e71bcd4b108d3,f9e8358d-0216-4351-a4cf-8a1b57e1b515
1795180160.463031,warmup,201,0.07005786895751953,81fe6f93c621412393a7521026fb86dc,de99a316-38a4-4017-af7e-e10e6a4116b0
1795180160.705324,warmup,201,0.05199694633483887,7e9577a3bf80456a900e1745534ab5e4,22e6878b-31a2-4364-a94a-54f9daa82365
1795180160.956168,warmup,201,0.054311275482177734,88a73eeb3a41494688161931b3f2d0f2,dce46916-4572-4df2-9bdf-70a7b940cd2a
1795180161.206967,warmup,201,0.08410191535949707,acf2f4457b724ab5b3da4ab0f57a4c3d,09ef6df9-78f2-4502-80ca-672166481b98
1795180161.457957,warmup,201,0.03756880760192871,eeb99aed3aa2474682bb942249b73f96,de89ee75-a4a0-4f50-b7a8-e305cf43126d
1795180161.707008,warmup,201,0.023017168045043945,a9263d7d93d04c818df6021d471c3fff,3ec8e5f8-2f11-4cf7-a965-337078d571db
1795180161.954315,warmup,201,0.03162097930908203,24367be67ee24f9dabb51672dbc58e04,25172e43-db3c-49a3-8294-187ebc18c469
1795180162.207732,warmup,201,0.02358698844909668,de045035ddbc4759a8ff1e8c03e62657,819465ee-7403-43e3-ab30-10db4551195a
1795180162.4623191,warmup,201,0.013826847076416016,bed4c0d33e864f52a18dae225d33ed50,b13abc97-d995-4b5c-9749-4e85892946c0
1795180162.7049441,warmup,201,0.011426925659179688,37bcbf41ff8a48939d46728863884410,90f819ad-45b1-4e79-b583-45c26033f52e
1795180162.95575,warmup,201,0.013515949249267578,f7c1e2922ab84f149cc615144d947801,e9f054a5-969b-4714-b3e6-948247656538
1795180163.2059221,warmup,201,0.014194965362548828,7e0523de687c4c358f88f591eb45631d,d708f6f8-6747-4a1c-82a8-7056048525a4
1795180163.457537,warmup,201,0.012165069580078125,248b9bec4fe34a5d991fd6246acf0c22,13bff701-eab6-4c76-aebb-1b4fe929d1b2
1795180163.704688,warmup,201,0.010187864303588867,73f64cbad982461cb68e042fb8b2ef6e,5247ee5c-efad-4a2e-87cf-e955a9c39ce1
1795180163.955895,warmup,201,0.01818704605102539,c6c1a0040a2f40d1b729f6e0895f15e8,e5a1c9b2-68fd-4c1f-aeae-4450649c1556
1795180164.20519,warmup,201,0.01907491683959961,00e6a3e5c771484dbb237cae0fbfc09d,bc2c0346-9643-4d27-a6f7-689b1bed3e85
1795180164.4559622,warmup,201,0.02111077308654785,4665df73d5364e5daa3a441143b7c127,a8594acb-4d13-4686-b53b-0346cb860fd5
1795180164.713967,warmup,201,0.014435768127441406,e1c3c6397f7f428b85cbcacdb1098b80,b07823c9-ee89-4dbc-ade8-321d3974dc56
1795180164.96303,warmup,201,0.021161794662475586,26d86cd296754a089f91b90439e83f60,8ac96c25-a31d-408e-a3fa-b2e6f372a454
1795180165.207382,warmup,201,0.025116920471191406,09994b987d8e47898ddad4fb10a09a38,e0d41a3f-d634-42c5-9496-bbecbbe7d733
1795180165.463873,warmup,201,0.015794992446899414,df1d6019189446f09582a5bc32757299,7f49017e-5ac2-4521-9772-74d2e55ff1d9
1795180165.7129421,warmup,201,0.012481927871704102,cfabbf97b3c14b02bfba1a74acaf4310,add4a145-d07a-49d7-8cc1-5f9774f6d5bf
1795180165.961084,warmup,201,0.019973039627075195,5b48ac86de57401eb7f3991cc1904749,7a498621-f882-474a-89a8-486990c9ed2e
1795180166.2129421,warmup,201,0.010935068130493164,803f2dd4c81146fd9a5d460b58a5256a,f20ae48e-27d6-476e-8207-e99a9e68019a
1795180166.454031,warmup,201,0.030212879180908203,b40cfbdfebc044e3b29b3621d95f536a,3df1f455-41e5-4aa7-bfef-8754f1eda936
1795180166.712905,warmup,201,0.02558112144470215,00b1bf8353934579a8a95b922f1fdabe,ed4d2d9f-b932-40a3-85e5-a9dc6a613e9e
1795180166.9541728,warmup,201,0.014276266098022461,9425018ce8324791b8d6eff29d7ee14b,9cfd6c36-ba55-43a4-9d51-13273a3a80df
1795180167.213747,warmup,201,0.08199000358581543,b4c47ffe312b4aac87827583c0843fde,a414a75c-0936-42af-8715-7013f1e94527
1795180167.4605298,warmup,201,0.015784263610839844,84d6ce144a2c40cb95f5ac7690fac5d3,1b856fcc-ff52-4147-90e1-cd94791f7941
1795180167.7135592,warmup,201,0.026343822479248047,b7939325d6c34ecea0d35c5d744d3bad,6b28b39b-0f68-4b90-a51f-1c152c58a6c6
1795180167.962919,warmup,201,0.015112161636352539,cd5cb0616a7743448c76ce7c9d9b314c,72dfdf21-f5e6-4b74-a1a0-e977ed622969
1795180168.2039711,warmup,201,0.011447668075561523,1166a0cbf6474c999930bda4cddd6116,02ab9f26-5ba3-4201-9c58-15aa457967f7
1795180168.458719,warmup,201,0.013019084930419922,cbac190c2cc744c1bef9610fec68ddbe,c762f7bd-4ea6-4f4a-a59b-fcfa20f2d405
1795180168.712939,warmup,201,0.03552412986755371,f4f37c03503344db9ab375351a65cd3e,255cef6f-545f-4ae6-ad6e-99ab8807e772
1795180168.957678,warmup,201,0.043527841567993164,b17d2bb8e0714a2d901e1f18637a2814,083f8307-1763-4a1d-b309-0d8d702e5060
1795180169.212994,warmup,201,0.0400240421295166,69ac03cd26df493790cfe98a041d5ef6,d0cbec89-cc67-4617-a0d0-26dddb89eae5
1795180169.462979,warmup,201,0.01599907875061035,2790773654d84f8d9887e150c9a7d7b3,e67158c3-4809-41f3-b959-9f8c8b06ead7
1795180169.712948,warmup,201,0.0437169075012207,4c0ffa1b3d414b4cb5c22e9ce7383a56,58d0c598-8d58-404c-9cea-d3a109fa3886
1795180169.9544451,warmup,201,0.20522069931030273,a5a0cfd85436458187897f6cfb92e828,4113d976-439f-4909-9baa-f550c1991184
1795180170.2053828,warmup,201,0.044216156005859375,f94451b87409492ba02e70fa8f3877c9,9c45f131-93ff-4678-9ac9-1857ea5116e8
1795180170.456413,warmup,201,0.028231143951416016,40ee2a373b2943918e5d19b44780f7c2,b8d0315e-40ff-4b20-a7c4-10eb8a89abd8
1795180170.705407,warmup,201,0.14519500732421875,3fe37ad54f6449628ea8ae3b55fabda5,8c124926-7f46-491b-9bf6-9155a58e33c1
1795180170.957385,warmup,201,0.1566169261932373,bd234bbdd7a84eef91106d8bb06e4d82,43c16843-8556-4fcf-a412-7c9c85de1829
1795180171.209719,warmup,201,0.02808213233947754,1bc7c13575584cc3a996d8e7d4e7130c,f59677d7-0cd4-45d7-8bba-4843b1a3b675
1795180171.708301,warmup,201,0.03726792335510254,eb4cff5c04cd476db79a908172e69ea4,26149134-587c-4036-8f48-f7e18ef4fe50
1795180171.464971,warmup,201,0.3508949279785156,a9a6b04df882480c9bacfdc2a07216b8,5f0fd18b-a430-4f50-9076-e8f8ab1f047a
1795180172.206183,warmup,201,0.16024208068847656,7855481b6d0541a087d7f15273a8b09a,247ddeec-8837-4c1d-a60a-2546978fd953
1795180171.957116,warmup,201,0.6614320278167725,0963b0b6357f44278d2c3cdfb17e4250,693d8bf9-8e2e-4efc-93ee-be9cbac0b8b4
1795180172.4628391,warmup,201,0.2640187740325928,18253581a2bd495aa28106a53393968f,c41b1cfa-02d9-43b5-b055-f014f7f57802
1795180172.704912,warmup,201,0.03334403038024902,e3a735c71221445fa876a3e44c9cfa47,2ccac5ea-d3ab-4bbb-b063-3b7c36c3ea4e
1795180172.9577332,warmup,201,0.06759285926818848,af4bb1940dbe42538d232d83ca9812cc,cf812735-2489-4e04-b672-b6143c2c78d0
1795180173.206496,warmup,201,0.024734973907470703,70f1582dba2f421ea1980dd833dd5a9d,22898e79-875a-4898-a300-845aa4a6d199
1795180173.455349,warmup,201,0.03874802589416504,e858bf8007064349bca1e03aab5eb26f,27cc175e-827a-4188-a728-b78cbc3967c9
1795180173.7049768,warmup,201,0.0689091682434082,a066bdd03a4043f3ad62d43d306acdaa,c07f5205-97d4-4f59-a9aa-8d05bfc1a52c
1795180173.9543931,warmup,201,0.018208980560302734,2e50d32ac368413cb7c4f82c97b55bf1,be9067a9-5d8d-4efb-bb8b-8077e902078e
1795180174.2129998,warmup,201,0.012910127639770508,2298a50853b442cf908afff6cd410dda,51aa7305-711e-4137-90b3-34a233cd9587
1795180174.46289,warmup,201,0.011858940124511719,fa1e496a8fa54b3bb1168cce8e1a4f23,9f1088f8-a141-4091-a877-fc8d93834d91
1795180174.708926,warmup,201,0.013499021530151367,99f56b141a6d499aab6492480f6fa0df,d5705025-1ab9-4976-8af8-bec4a632f7f6
1795180174.961039,warmup,201,0.05283308029174805,c446fb3191384b2fb20905a33ca1259d,7b9d27c2-7bc5-4768-bbb2-7b433f668401
1795180175.2130811,warmup,201,0.046158790588378906,6cac815449fc43ba9ee764b2d3ea89ea,b44e4411-77d1-4acb-a6f2-f94fb6b639a1
1795180175.459479,warmup,201,0.08919286727905273,00672bb9be9641269cac74d9054f8f7c,27f04e7e-a0fd-4f67-b93b-32560341c85b
1795180175.713186,warmup,201,0.07668614387512207,909316d4377046908d3d9b80c5f63734,6c14d788-4b33-4886-8fcc-b87c34433962
1795180175.95504,warmup,201,0.02902507781982422,e11701b75c89461f9cf9eb4c49119993,34580b41-aa71-4bf8-935e-4b661f85326e
1795180176.2096431,warmup,201,0.023485898971557617,94dca22c54ce4adc9773397279aeeb94,4778e745-e0dd-4ba9-9739-23f228358723
1795180176.454023,warmup,201,0.013267993927001953,0c731c2d3c904251a1aaddc02936ac72,b31feb95-00d0-4633-bef1-d3a80d59e132
1795180176.712258,warmup,201,0.00846099853515625,5ef7d7287a94477fa7506a8974c8e073,9e76b067-3773-4042-b008-3bcc8ae37289
1795180176.9543521,warmup,201,0.008093833923339844,fc63470f48ff40cca32ea6bbd1c87df6,0ef443fe-2d88-4098-828d-5dcd08f0cde5
1795180177.204947,warmup,201,0.010506868362426758,f302b68d4cb641eba2ade0caa0d5e7a5,d11c40b9-87d3-4c6c-942d-f0cfb14c72a8
1795180177.462801,warmup,201,0.014312028884887695,d62ebff655b64a349542bae704bc43c3,6a603f9c-232b-4fb7-86d2-4d762a4e6e89
1795180177.70669,warmup,201,0.011358976364135742,b76a61a69f024f979a61e9ac4206ba23,d252996f-f60b-436b-88e2-5f3f5a19774f
1795180177.958333,warmup,201,0.010645866394042969,f81e9d7fe46e4d3f9d2c629a64cf0532,70b74b69-55d0-4b4e-8a63-09f3ca44988b
1795180178.21154,warmup,201,0.011842012405395508,d927e2354c124bb1b133252d7ca78ec0,59f1a68a-dc24-4ec1-a940-18e8c75c5544
1795180178.456432,warmup,201,0.016075849533081055,fab54f7549c84c32a9333b523bbaf313,4c301fd0-c4ef-41b8-b803-2d0f3d4a47fa
1795180178.705439,warmup,201,0.016112804412841797,6795365d631349b8a2b6b0f020fe56d8,f60cab03-ca55-4f74-a133-c5c8ed18c7e3
1795180178.957304,warmup,201,0.012420892715454102,76ce4d9cb2d347e7a54f347e1555059e,525440d2-b4bf-4250-9b7b-31e89fb57a27
1795180179.21025,warmup,201,0.008681058883666992,d6b683ecea3741df86168a9ca995ea9c,cd03fb05-c3b4-4b57-8151-c5e749cfe2c9
1795180179.461452,warmup,201,0.015002965927124023,cba9638961a040e3bcb4370420e5a858,f7facafa-db9b-4922-9dc9-e65500737f75
1795180179.712847,warmup,201,0.015548944473266602,bf553ad417b040419ae155120de940bc,2aea59d2-0e2f-4929-9651-43073423520d
1795180179.960278,warmup,201,0.01565384864807129,ef20979209bb4e448255f580b1a07b61,c20f1a67-e981-4473-b1e0-422fd88447e7
1795180180.213103,warmup,201,0.015707015991210938,a32354c47bd845f5bfeb45c98f03c848,9334f89b-289f-4d50-9b87-0afd45cd05aa
1795180180.459705,warmup,201,0.012166976928710938,f0fcbc4e2d4b41f098e3429bb23ca236,f856a16e-3469-494a-afd7-943339c4956c
1795180180.713276,warmup,201,0.019527196884155273,93e6ed25d7654a8a8aa5a6191c03ae82,a5c347ed-5b9d-4df2-836a-29ddb2d05126
1795180180.95435,warmup,201,0.016804933547973633,edae63a4317c42b4a2afde5303f1871a,efbff025-41ee-4761-bf4d-3dbe23d4007f
1795180181.203885,warmup,201,0.015861034393310547,4531a8464acb49d789e97b7d2cd44f74,79d25593-3116-4f7e-b555-01dacf572512
1795180181.4541068,warmup,201,0.010253190994262695,b35a9cb2393442ec88077c5277b69223,cb921d6a-632f-4966-9259-46b8ff7a936b
1795180181.712745,warmup,201,0.04211616516113281,b55996e8ad384e92bd22553058887531,0befa854-1659-4b69-82ee-2d9c4452c56e
1795180181.962806,warmup,201,0.014417886734008789,eb54c368418c4ea2a81cc55c18f33674,d1339fe5-4fcf-4c51-aa31-ca0d96a21846
1795180182.204462,warmup,201,0.012550830841064453,27545433b2c64c5585b1f857b04c41af,2cae5a6a-1c7a-4f5a-92e8-e31936a8deaa
1795180182.462759,warmup,201,0.013756036758422852,ba01f205cd36452f91acdcc144693ba3,3ca8359a-17f4-4a02-809a-a0b3258c53bf
1795180182.7060359,warmup,201,0.016193151473999023,c9446e7e7bbd4435ac4dd80d99da6237,18f5e9a2-e710-4942-80d9-55604faeef13
1795180182.956501,warmup,201,0.022050142288208008,5c19e1f85a624e03a79ed1bb9d338b27,0aeb8bc4-2242-449a-a655-4cbd016f1ee1
1795180183.2053308,warmup,201,0.014279365539550781,b35b8690a19f44459906848fc8ea13fb,53bb3ff2-cd49-467c-b323-73ae30e5dafb
1795180183.462892,warmup,201,0.05702996253967285,929910e78a024ae092c58d12a1852b24,14309d64-5f24-4db6-b641-3c5e031e1204
1795180183.706051,warmup,201,0.03141498565673828,a4e93ec239574d57b20f10bd5e7560ad,b6771803-b769-4d57-8b38-1ed6fc6eab85
1795180183.960104,warmup,201,0.19160008430480957,646d23e10c5b44f494eb4538753c608b,1bf69a09-4231-4a27-9bfc-6f728053852f
1795180184.203552,warmup,201,0.04222893714904785,577e8b16c9cb49b4828a8dd043e749ec,5e8e8000-b2db-4d94-b1c8-0a6636d678f7
1795180184.454276,warmup,201,0.014716863632202148,27fd975e87d24480ad32d1ff15d95cae,24f15864-5d6d-4d0c-aed0-8afba06f22b3
1795180184.707995,warmup,201,0.03830385208129883,8c2abf5443824c13b1d16a6b7ab8fefd,2cf6a6cc-2a28-4a5b-8c77-7228fde62e7c
1795180184.963094,warmup,201,0.01816391944885254,4db1a0d088e549f6ad9b3fe4e3bfca7b,8d0d23ad-6b5b-49f8-a967-0f81fbf04fa4
1795180185.213154,warmup,201,0.011205911636352539,277ddd9fe9bd4c798830e9f0b690aa58,940bacab-a245-4c62-85dd-cf7dc89773bf
1795180185.458055,warmup,201,0.01757192611694336,89a9630e6b3f4accaa02a5e4fcbfa9b2,0e2796e4-eea9-441e-ba7a-adc3318c4bcd
1795180185.717603,warmup,201,0.04112505912780762,0c81fd8100a644d886ab797b31454edf,2a6d1be5-813e-4431-8acb-de28bd6cc485
1795180185.9632158,warmup,201,0.014150142669677734,f726becda74545468eccdc64fe022e3c,3310626b-3ff1-49f9-85a3-d46b310862ec
1795180186.2084239,warmup,201,0.013811111450195312,ea4985ca5f3340f1878f7790e1d7dee3,bb9f5be6-eee7-4fae-9d7d-708400a5f332
1795180186.454124,warmup,201,0.027569055557250977,02b44be192d646bd94c431a7e0cb6ce3,d57d83f0-2dae-4fc2-a1ef-988fe1f370f4
1795180186.713099,warmup,201,0.011459112167358398,0dd312b7244a4d5887379d5c0a002e66,6a1c1ae8-f2ee-4851-9b63-6180c5b7b7ec
1795180186.956599,warmup,201,0.019530057907104492,92341c3304f3480ca74f390353e35541,54f84857-de49-4ba2-a228-0737e5c4519a
1795180187.208894,warmup,201,0.12175798416137695,624595d09b0b4341a6ee3562a5acedc3,c9526170-ffc5-4155-be96-f4afaf01ab28
1795180187.4532921,warmup,201,0.030305862426757812,76f2be0fd53c43929d5dbd13c8ea9f4f,a14968b7-2b52-4296-8a57-dc2da3cde3e9
1795180187.7090461,warmup,201,0.021664857864379883,b8aa6f23adba449cb738ddb921c834bf,cc87f9ee-79d8-4bee-8afa-6d9bdc0e50a0
1795180187.956699,warmup,201,0.011160135269165039,fb8bc2ed80e248a180abdaac7d680b97,c67baf4c-da95-4b1c-8fd9-54759dd87f7e
1795180188.203029,warmup,201,0.014909982681274414,6b91e855e5004849aaa0d0b611cf98f7,44162c07-2165-4353-acea-cbdb32f1f389
1795180188.4538748,warmup,201,0.04066634178161621,94c834f81c084b91938fab2d21851366,fb7693b5-7fbc-4675-92cd-3952ec2ecb97
1795180188.704454,warmup,201,0.018157005310058594,d759a840803945fe9a93443f2fdd0b2e,4444f578-97a4-4aa9-82c9-a1bd51b35dd7
1795180188.963008,warmup,201,0.013981103897094727,f57660a0e9604d7a825a6f5e3743bd94,8577c34d-68b4-4ffb-93ac-4b7bd4c83f21
1795180189.203331,warmup,201,0.023202896118164062,244114a321374316adc578862f9200c5,1323a0aa-0302-4608-87b8-55451b318c97
1795180189.460938,warmup,201,0.015442848205566406,693f36b531444b85a5ada1bae9069a5d,65fc0dab-bb6b-40e6-9d93-218728ef2118
1795180189.7105951,warmup,201,0.010252714157104492,775511c278684cffb821bfdbdbb40ff1,082dea81-4649-43da-af39-8148f5e3e17f
1795180189.962133,warmup,201,0.027779102325439453,704f62a681bb4d1d902eeecc72dc8c49,020729eb-6c61-4562-ab6e-4261cbb48cb7
1795180190.208668,warmup,201,0.08618903160095215,d58246f4898b413cb3a57efecf51996e,182f947a-cffb-472b-88e4-ae63b9055269
1795180190.4556952,warmup,201,0.012355804443359375,85c0ca16930f4f209da3a945ce5c4c0d,48f24757-6028-41af-b1d7-8a4f16f0c467
1795180190.712575,warmup,201,0.021033048629760742,8b5ce8b40b5c4735adb4be19e104a00e,75758c4f-8b76-49a1-bf16-396c6d6f074f
1795180190.9550889,warmup,201,0.01253509521484375,20981200b4c649a2b3eb03ea7d2953ff,259fa757-9ecc-4169-a160-c9c5b765a199
1795180191.212717,warmup,201,0.009023904800415039,18e3ddcd29164515b49ae4df4546ece9,9cd61279-0edd-4c2b-87ad-7da34a443709
1795180191.462502,warmup,201,0.01763176918029785,e28e7f1f48144770af9dcffb74421210,272f31c8-dd14-448c-a2b0-0232ead05ddb
1795180191.711782,warmup,201,0.015043973922729492,a887526046214c9296d8d7abe36313c9,004511dd-d814-4b80-9979-1e68dfc12f8f
1795180191.963177,warmup,201,0.020080089569091797,6e0ffd38e2224d7fa79adfe0bc392818,900a372f-3281-4637-a8d7-8273d340d0f6
1795180192.21304,warmup,201,0.03302407264709473,d3b15ced1f164a9eb8b0af6e883a5d0c,5ef64385-53b9-4f95-ae89-866084089856
1795180192.4593432,warmup,201,0.026756763458251953,288161a679294cf991c8cf1f700bbee8,5631d6b2-9b07-4d05-be87-8dc2b82dd2a5
1795180192.706445,warmup,201,0.02288222312927246,6ff73f860a2d47ada41da4ef9b9d8c1a,11733eb0-f5ef-4e7b-8667-092abf95794a
1795180192.962646,warmup,201,0.021658897399902344,d12c7188544d47e6ac8491fab039b363,ac8ad5fd-c7c8-4373-b22b-9a8630e607da
1795180193.2061381,warmup,201,0.019336938858032227,075af4e320e344da911d37c4c5a34a91,d54c1b25-df52-4413-a8bf-c00e8fd67c7b
1795180193.458136,warmup,201,0.017496109008789062,8c165225b2d64d238b3ad2bd4a696c25,e7fb5585-af0d-41f4-a0b9-a14066005edb
1795180193.705204,warmup,201,0.022547006607055664,9599bf2b238a4c78b7a8fda52bc67ff5,e7bb66f1-582a-45c1-b29d-aa4dfcf01619
1795180193.9539452,warmup,201,0.26253581047058105,cf4db02bb778433f9ee50fa7fa47cfd5,cc18753c-758e-4be2-b511-9ba6da8872d0
1795180194.2069068,warmup,201,0.022910356521606445,cae9dd0a6bf442a4b8e67bad0875c97b,958e8fd7-07f0-4812-8871-ef48157cf903
1795180194.460042,warmup,201,0.25853991508483887,885dd8b642104c1fa04f84688d816fc9,f9b1e565-f8e6-4abc-9e7a-65a1c9e4a40c
1795180194.7131689,warmup,201,0.07060122489929199,1c56908d205741b2964f718ddb5f3def,69258830-947c-4245-a833-78176422fe70
1795180194.952873,warmup,201,0.2982909679412842,8ac2ceb1f42040f8bc6e894fe13abbdc,3dad5269-9754-454e-b9cd-264f86774850
1795180195.4555662,warmup,201,0.05453991889953613,3592f8a05103400fa96fcd57e0f5f7c1,c078f8f6-f584-43fd-85df-acabc6b876a9
1795180195.207063,warmup,201,0.950869083404541,80b7746064cb43a2adc4f588a84601e5,f0b4607e-5350-45ee-b07f-2be09e95fd99
1795180195.70959,warmup,201,0.4569091796875,7b2995e04de8452db02d42aed9eaca94,4cda4fbd-ab10-48bd-88ca-3c48af14315a
1795180195.956768,warmup,201,0.22559404373168945,780d3f933edf4e08be22421f5cbb4162,39d0a6bf-c992-4cd2-b4e5-f7796a042a6b
1795180196.209187,warmup,201,0.05438494682312012,f5ea813035874d0fb0b08e826ead91e0,fea354b4-7a1c-496c-a804-5e572c0767f3
1795180196.454772,warmup,201,0.04239296913146973,bd79b9a297a34dc49a4d6a3b1cba46c2,6a63f12d-a5a1-47d2-bf37-813c326442e3
1795180196.706481,warmup,201,0.02643108367919922,615b66824e97490bba5193a10df8fb5e,ccc105e9-ebbd-4f96-ad46-85d2ae34e259
1795180196.953847,warmup,201,0.024856090545654297,1ddc5ff0885040069f2ebfcbe03bb60c,5d94b5f8-2496-4f7f-8fe2-23c4975a3874
1795180197.21262,warmup,201,0.015910863876342773,609bb8bdd431457186d75aac72c48f86,9b98b6b1-31ce-4825-b220-c92fc034d556
1795180197.455989,warmup,201,0.03686928749084473,f91f41fc62b54c05b486321f29266270,0c44435a-4ede-4ee1-b83b-4f119384dcaa
1795180197.703308,warmup,201,0.02806401252746582,42ecb8cd11ef4e60b95a28d839f19903,8eaf59f5-d4b1-4079-9162-87292d73f31e
1795180197.962825,warmup,201,0.012208938598632812,2411d19e7ea942ebbe8d97ceec71bc0b,80a79fa4-e59c-47c1-9048-5cba540a330b
1795180198.2126641,warmup,201,0.013143777847290039,faac45e8c35b4e22907ec7a29991e111,f9de74b6-00a9-4736-b224-e50f0e07fca8
1795180198.452794,warmup,201,0.010195016860961914,216c854ded9b40ad84c29e4dca3fb40e,8b8b2144-eeb7-4bfb-bcf6-1c3fbd1d3500
1795180198.710228,warmup,201,0.01584315299987793,bb930213cbeb48f4be1c535416c3ebef,63588c1d-2d95-4935-9daf-4b6591f718ac
1795180198.955477,warmup,201,0.022932052612304688,98735988ba83477c8dc8255e39c6e9be,4f26cb6e-f5e0-43f5-9ff3-00410d4fcb0d
1795180199.20573,warmup,201,0.027022123336791992,84770b0f61c3443b9c5b95d8dca08be8,44c5c309-221e-4296-a57b-5a2624514949
1795180199.455035,warmup,201,0.0162200927734375,a13929fa98d745b6badf08b6afed734f,616a1684-6a86-455f-9c35-98d63f88a237
1795180199.7043462,warmup,201,0.021170854568481445,1a22338792844a6995025059d1573d8b,3fd685cb-bb6c-4b53-a2f2-e67d66e52ecf
1795180199.9531882,warmup,201,0.013972997665405273,1803c2fcc5644cd5862fa1c5ba572c5f,ada74ea3-81f8-40b1-bb49-62b850483cb6
1795180200.212534,warmup,201,0.016909122467041016,f0eb8bba585640e59b164fa58243df6a,95fd836a-e610-40f7-bded-88cf8bbc4d9e
1795180200.4543731,warmup,201,0.011936664581298828,cb40183ca2e84009bb204da20c980af0,172837b0-53d7-4056-a9a0-6034f129a3fa
1795180200.7096949,warmup,201,0.0234220027923584,399c3e459be045dfb2bae117f1ce2dbd,93cd6edc-c047-4411-befb-6e07e178e52f
1795180200.954174,warmup,201,0.02118086814880371,c9fa117ad07a462d84c7b43f8922716a,f5fd669c-648f-494b-b836-02a6e00fc0dd
1795180201.203704,warmup,201,0.02590489387512207,b23bb2e856984c60b6c634c3f5a1bf8e,231408c4-a640-46ec-b694-ba7b7b26808d
1795180201.4537852,warmup,201,0.012441873550415039,293dfa83cfdd475ea7466ed8741d856c,eec778a0-e94c-4fc8-960f-98d89e79160a
1795180201.7075672,warmup,201,0.009174585342407227,46c8ada6c78149efa0a7ca8e4a5116c2,cdfd8d19-fb43-49dd-8a9f-2ebfec681b55
1795180201.953985,warmup,201,0.014340877532958984,fab2d1b470a04bd59bbd3ab89449a39c,5338f15e-4c79-428d-aed7-abb03a0d2e38
1795180202.212538,warmup,201,0.024313926696777344,53fac7e2df3e480db30b3d7249691ad2,04d13d7d-12fe-44ec-9185-84397ec920e3
1795180202.454725,warmup,201,0.011285781860351562,77d6d1070dac42308a797d848af7d655,7f7e9623-4f82-4732-913b-f5ef7419906c
1795180202.704545,warmup,201,0.01698780059814453,d1eca4e404474f548e4c9433975ff7e1,37f34032-000f-497f-9a2d-af4edea502b9
1795180202.955153,warmup,201,0.012134075164794922,b3ba6e15f8464b80a44e6b201195a5ce,98fa1948-dcff-4dca-86e4-028ee0edba3d
1795180203.208073,warmup,201,0.04192614555358887,a3100e87452b4ea6aeb47bf91637d1a7,d334cda7-3c90-4805-97e8-64b8e2f70a1b
1795180203.4534369,warmup,201,0.019451141357421875,42366783be314ea1a699ee7b04915272,882e0ea6-0a04-47d8-84c9-535ff6ba8cb7
1795180203.705002,warmup,201,0.019451141357421875,cbe35d7bf653422fbed291fdac71b9d8,fbad4794-3478-4d93-b051-1a7e7aa973ae
1795180203.9624739,warmup,201,0.020875215530395508,ae17f67d661349cc8def40a8ef9c50ef,78d6450e-c4ae-42b7-b8d0-e84b45063417
1795180204.206752,warmup,201,0.013983011245727539,4cc9c8169d364dce9be1b8f554f1b904,a22865ce-7d25-4d11-80c7-eaab1be90dad
1795180204.462346,warmup,201,0.01632404327392578,e7fa08a3cfce42e590897ab29d406ed9,f0a8ceb3-5b07-4e6d-af9e-b268b1ae4f6b
1795180204.702373,warmup,201,0.027428865432739258,7fe94400e2124e1f994f4ceee5a86b61,fe1fd31b-25e5-4426-abe7-47d20a216f68
1795180204.9624069,warmup,201,0.022263050079345703,1e442c69b70d4c91a2367e58d7c85152,410322ca-61b6-46ad-a236-3403bfd0304f
1795180205.2122922,warmup,201,0.00986170768737793,e38450ee6a9843288edccd96a9073daa,d2102740-3b08-47cb-bf40-a3c5a083d68d
1795180205.4595919,warmup,201,0.012141227722167969,3549a7e31a724d64a9838531e9858237,94382e93-ef4b-4521-b721-fd16cd9a0aa5
1795180205.7061708,warmup,201,0.06794524192810059,c7972779afde4c389d99a40a0a1ade6e,7da588d3-2ad5-4b84-a5e2-c5bb86179706
1795180205.960969,warmup,201,0.017190217971801758,421aad87ea14472d9fbda6c3dd9ba7d7,4e603bc5-dcb7-4a97-971c-d11de4f95acb
1795180206.2061582,warmup,201,0.018009662628173828,d1edf937f3d34f888a97a825cd4349ce,004ce182-0fac-4730-a3f9-70822af82d14
1795180206.453884,warmup,201,0.010658979415893555,d760aca1b09a4234aff7b454c14917a3,18dabcdf-5de9-4714-82a2-15c3173e9eec
1795180206.707329,warmup,201,0.009528160095214844,33ce61e42af349f9bc78467bd915b819,5ef9abc7-bf6e-4bea-a622-f4a7527aaedf
1795180206.95679,warmup,201,0.01139521598815918,739212b4e75f419fab41b1eeec99ac92,16cdc8c3-d46c-47ea-9b71-6f75b08d15ab
1795180207.206121,warmup,201,0.0551609992980957,7ffc640f460d4663a2828f833d49a850,a15bc994-c880-44a4-b584-4af86433b876
1795180207.453679,warmup,201,0.0324397087097168,a7f33fedba77468e891995294eacbf6c,ff7691bd-5d00-42f0-a281-a1268e2795b6
1795180207.712791,warmup,201,0.020277976989746094,122ada18bd024ac39b6d09e8a124ba41,351ac8aa-6f70-4695-a389-e20c5c9d6767
1 timestamp phase status seconds trace_id message_id
2 1795180069.9570692 warmup 201 0.16097378730773926 e776267093834c7b89d8d53fd3c6f8b5 2d1f4e3e-8b85-4177-a48b-657ac153f43c
3 1795180070.212939 warmup 201 0.043734073638916016 eb474a2a92eb43989227a6f21537ac21 8afc44f3-b628-47ca-a48d-c38434e6d664
4 1795180070.459362 warmup 201 0.2732198238372803 0ff25933db8c40428275ec65a3905bf8 b3d8b459-e5e8-4e85-be31-8ca4aa62b4f1
5 1795180070.712351 warmup 201 0.035295963287353516 cc60cc8a82014effa9f831f3ad2909d3 bf276116-3268-4256-a3ce-4f4a8b48608b
6 1795180070.957216 warmup 201 0.06459498405456543 6acdfc552b2242458bb20b805a12bfdc e45fb1d6-eddb-4636-b765-d3f1b54ef47b
7 1795180071.211649 warmup 201 0.03757905960083008 96156e96fb4a462b919edffb002c1277 ebe8451e-7aa1-456c-87d4-d770b07e4baa
8 1795180071.460105 warmup 201 0.009978055953979492 be2e4d671bc646e3b0ee2b882d6dd4a5 0255c592-503c-4046-9624-35e44a618fc2
9 1795180071.717076 warmup 201 0.011640071868896484 c877e6f6a0dc4b0395715916ba76a283 fb9c0ad6-ae59-4cb5-9a50-df03f5fa5cc9
10 1795180071.9642081 warmup 201 0.020227909088134766 0fdaa448d3964c5e9a44469c6cb68277 4a96e606-549c-4427-baab-191bb0d36e5c
11 1795180072.21425 warmup 201 0.038519859313964844 43b80ef7af8044e9b2b1bc1234c15f13 8cd6c3e7-6451-473f-9259-1aca3d8ac834
12 1795180072.460448 warmup 201 0.04502081871032715 dbf31c750923429095b0196c95454728 56503f1b-d7c3-4849-8654-d53bb7af63f2
13 1795180072.7154849 warmup 201 0.031109094619750977 46f4da900e4a4e009656802df64f2c8f 91d88223-fdec-4c91-bca0-4d5b28d76604
14 1795180072.959334 warmup 201 0.02359318733215332 99017fc0c3804148954a692be6c93b41 5579e52a-c91a-46b1-ae3b-b0475bc94130
15 1795180073.208265 warmup 201 0.027540922164916992 316c362b4f4e4ccfbcd7ab3f211f8cfb 7139f79d-c498-4056-afc8-e5c1eaaaf9be
16 1795180073.45748 warmup 201 0.07488799095153809 feffea2172e7461cb4171f44ef34b361 a1f19e0c-0ff2-48c3-882f-dee7064313f5
17 1795180073.713435 warmup 201 0.015566110610961914 2547a0ea547441dbbe974d44b458241c 9324684d-fe9e-43a1-a84b-bae5523663b9
18 1795180073.9612842 warmup 201 0.07160663604736328 00228b48dcbd4a1c84e81aa4c977d016 0f6ea533-bd56-4ed5-8c78-dab41ee30320
19 1795180074.216365 warmup 201 0.032988786697387695 e807fef317064c769b2074bdb344bef4 7b5834e4-65e0-404a-b07b-a58346afb77d
20 1795180074.467131 warmup 201 0.02714705467224121 c8eab445c70342b78ab3399b605b83c7 c530b3a8-c5e5-4c69-a3d9-0f0a8e20eda7
21 1795180074.713984 warmup 201 0.026133060455322266 7e6544a5661349c29edf9af88d3c9fc7 3dd3ed27-4056-48ba-9e8e-21515de5d2a0
22 1795180074.967433 warmup 201 0.023201942443847656 9a9720dfbe2442cea02dccf4eea97599 dcf1e137-2670-4a59-a022-58bf88cd9356
23 1795180075.213536 warmup 201 0.027930021286010742 59642b7fe13d4635833fafa6207876f7 5ffb0a8f-0811-4fce-96ba-fa8e5414153c
24 1795180075.459161 warmup 201 0.01526188850402832 bdbf4b92112e4787890cf366c3f67e71 a03d1041-2060-4a43-8fcf-77da2645cbfa
25 1795180075.708655 warmup 201 0.12127065658569336 9e6b038f957743939b2ad4bdb3e7209f 3e944606-3b6f-4131-92ba-7b70a748f85a
26 1795180075.965733 warmup 201 0.05122995376586914 9234b276a32f47a29b9fce67868601df 2b4bf449-2982-4d13-85f5-0a549a65f716
27 1795180076.21755 warmup 201 0.033921003341674805 c0d749b56e5049888a86e50212011025 90afb79d-6102-46b8-98d2-b7ff3914345a
28 1795180076.460042 warmup 201 0.017657041549682617 2984009087fd4f39bb9bb7cf1fca8a46 d833b693-bb00-4cc0-8aba-c116773024e5
29 1795180076.708305 warmup 201 0.009041070938110352 138f1cfd9f8845098388b0ee14118da1 0c474577-7e64-46bd-b965-80e9d7cfe10b
30 1795180076.963747 warmup 201 0.012278079986572266 03b2408cb3e344a49ab3f04c9607f160 6435d985-2856-48de-b33b-e335146db38a
31 1795180077.208763 warmup 201 0.025561094284057617 f545f350e2064ab697c70fdf8c38f2f7 7d2ece38-e7a1-4afa-a4df-74f9e0804e58
32 1795180077.46699 warmup 201 0.018079042434692383 3f245fa447364122ba33abe0f2c09179 7047997c-e173-4736-aa5c-36394ebe762f
33 1795180077.71347 warmup 201 0.013484001159667969 2d3da404a33444689cda5369dff039c1 c5c4835c-4401-4f50-9554-664126393837
34 1795180077.9609568 warmup 201 0.011887311935424805 9d82a52c79664331aa13a40571badfcc 1faabd95-bbd4-4d1a-8d11-410ba137b9dc
35 1795180078.209839 warmup 201 0.012413978576660156 57ab3eaf40da43b780d29f19843cbfb2 65896890-75bb-4934-8bdf-99ca1f783b40
36 1795180078.466974 warmup 201 0.012367010116577148 cea396ac874b472b937087e684fad26c d762933f-ef30-4869-9d23-f015363c6f3f
37 1795180078.7080152 warmup 201 0.01634359359741211 8574bb35d3d24cd1aec875b112e6cbf0 c6946cbf-85e0-4d06-927e-76f43486e875
38 1795180078.967406 warmup 201 0.017637014389038086 bfa65e35fcd04d238601a48a17b433e5 5cd65d42-15dc-48d8-b965-c0b70970e96c
39 1795180079.213855 warmup 201 0.02095198631286621 93c131a83a3049a8823296d254f8e444 772c2a2a-9d59-45fb-b235-3e487708fe9d
40 1795180079.466801 warmup 201 0.009394168853759766 3d7c6fb3c23e4a5c905c949efb549c05 fb4240c2-55be-41b0-a2f0-f0802bdc69ac
41 1795180079.713575 warmup 201 0.009850025177001953 6fc2e838823b4f56a5562957a968979f 2851c65a-763d-419a-ae13-74da0e76fc81
42 1795180079.9616141 warmup 201 0.09672403335571289 5677b6efb3034fd594a53a94fdb6e4cd 77c8df03-15b3-43fc-9a1d-386f9ff3e1c7
43 1795180080.210528 warmup 201 0.25554704666137695 c64242d9feee4b45af94b4a54297b725 b14682f0-d03a-4e5d-8e19-18f5544ef9b5
44 1795180080.459133 warmup 201 0.04615902900695801 5722b7e2ddc84215b9dfc1dfc63fefcd b241e81e-ac05-42d6-b38f-a1f034400491
45 1795180080.710989 warmup 201 0.036536216735839844 86b110dcacc7437f8f63805c9a881dc8 c6a1ad13-e162-4bcb-a779-8a819232e4d3
46 1795180080.95889 warmup 201 0.02017998695373535 ce58cb069c9b4455979a226de5bb77d4 14b20808-646a-48ee-a811-38d17f73eeba
47 1795180081.216896 warmup 201 0.012832880020141602 893f8fd075df4a24adbaeb3424dd842b b5b81a9e-9512-4427-8bd0-3f0fa5c7f6b6
48 1795180081.4668531 warmup 201 0.05256390571594238 2b3598c4da6041ab812aa3fad1f8878f c065a623-dc6a-4f4d-9614-eb13f97966bf
49 1795180081.7096372 warmup 201 0.014587879180908203 8546f1a24bd74842b29ee4600bad0e2e 2b68798a-8710-4d8b-a511-995c37de2613
50 1795180081.9634979 warmup 201 0.017155170440673828 925a99f02c5044c49e5c914e565b07f8 44b041e0-814c-4f00-8640-3c398c8b601c
51 1795180082.2102349 warmup 201 0.023682117462158203 165c55115b8649c380b7552bae3f579d 9b01fd09-ed33-461a-8595-911e17590996
52 1795180082.466957 warmup 201 0.023247718811035156 a6ca3d77dab14658b899fef8fc4f984c 7e017c78-4c50-44d6-a222-a47a48a43fa2
53 1795180082.712265 warmup 201 0.015156984329223633 913d65477bbf43e4810325d8f955aa95 42cc1f17-29f7-4e4c-b5b3-bba9a9fc2c49
54 1795180082.966871 warmup 201 0.014148950576782227 b2278b9f6d014d9daa209d4de38cf1b6 af0c15dd-80d2-499b-a0a3-1983237d6379
55 1795180083.211093 warmup 201 0.022732019424438477 c649ef237f904d6cabbad2aa6672cdc7 deda125f-e316-4889-b197-3f099963cc39
56 1795180083.465457 warmup 201 0.014335155487060547 7236502f12884b019f03115817e3cfc5 9ac3c065-5c39-4844-937a-3dbe092990b0
57 1795180083.716855 warmup 201 0.014973878860473633 d5573690968947ddaa68bbd9dfc54fd2 1d6465fb-1b33-4057-a73d-f3a95cfd5666
58 1795180083.962472 warmup 201 0.013889074325561523 27c7826e18a6438db0e9c29cde09d003 ac4ffbb5-54de-4b46-8a8e-d1a5bbb24725
59 1795180084.216266 warmup 201 0.010313034057617188 3d133c89500445569c772981d7a6d8f2 dbc8e815-7ef5-44da-941c-a6cccdb42a1a
60 1795180084.466589 warmup 201 0.027796030044555664 ed047dff153e4088b1f929221a33eaf8 f74d0068-af09-4a33-8d8c-8008480b2bec
61 1795180084.716776 warmup 201 0.020221233367919922 8a65f7dc12ab49bbb8605bd6f602be4e 17a6f33a-9636-46e9-abe5-1221c741b91f
62 1795180084.959736 warmup 201 0.02201390266418457 609086211d9044ee91129ca6cfaa07fd a534b4ee-e80f-441f-b2ef-768613aa654f
63 1795180085.208767 warmup 201 0.022257089614868164 ea4a411fb9c04271a5ad8e6b1f1168f9 e38ebaf2-6941-46a1-97d6-63d5202d4f11
64 1795180085.4657888 warmup 201 0.03608131408691406 dfee0690d5d749988c9bd39aa51a9ec3 1a6978d7-ba7d-4d7b-9b4c-6595bd07ab1d
65 1795180085.710897 warmup 201 0.010241031646728516 239f8a508e744cfdada3dd0492f45d77 1bce9275-d0a3-424c-aa80-f8d25483e609
66 1795180085.964003 warmup 201 0.015243053436279297 950f416c575744038378061499ba2b72 74641199-21e6-4de9-941a-37e489252e2c
67 1795180086.2077 warmup 201 0.021860122680664062 13da9df1a5814f4ca8da0cb28caeded9 c4ef5518-de9f-43b6-b05c-73a1f16f8cc9
68 1795180086.464135 warmup 201 0.04335808753967285 ffc1aa67328a4b648efee25dc0f80115 e3323242-ad8f-41ae-9942-bd64702c4fa2
69 1795180086.713995 warmup 201 0.013170957565307617 ff6993364f5545448fad2c42fb86f2e4 7b43da44-ea19-4ff7-972f-b48bd2b3ddea
70 1795180086.958281 warmup 201 0.01422882080078125 9def7c264a2749a3a0bad35a6cec362d 5d3391d7-c86b-4789-95b1-59414122b2b6
71 1795180087.211974 warmup 201 0.057511091232299805 abad7cc034f04605ae5e81a60de8014c 2bd49519-3a50-4495-bfdc-bf5dc787fd0b
72 1795180087.466767 warmup 201 0.010188817977905273 0c839d1587424440b7218b6581d0e536 ecc9ecbc-c3d1-4d07-9bdf-a4abb256aea5
73 1795180087.716816 warmup 201 0.03182816505432129 7c535a26c45341e4a586bd6ba39254e7 cfb59993-e9c4-44c2-b182-9b05ad6f15b1
74 1795180087.964291 warmup 201 0.014871835708618164 09482390f77944e6ab2c518c0e01bca5 b65376ba-6344-4203-98f4-4b828ede7d57
75 1795180088.2152479 warmup 201 0.014531135559082031 21374ca696c34dad8bd1743aa2acb6d0 2adecbe0-a793-4ac6-9ed7-d5d76b8665cb
76 1795180088.4627619 warmup 201 0.014763116836547852 3bc9b4a40c1b48d4b9ad9c7f78d1fe24 b87f059a-7ddd-4471-957a-ec8ffc1d037d
77 1795180088.709258 warmup 201 0.02199387550354004 b71531aab7b846b784ce55529938682e 825dca60-b7eb-41c1-b966-3708fd3c0519
78 1795180088.957341 warmup 201 0.012842893600463867 522ca628e4364233b9125e64327bc83b a0d28109-e2d4-4911-a0b2-d44a6e8a5ce0
79 1795180089.208671 warmup 201 0.009891986846923828 8020a9c2e82b4342a5864ea2da0dbe2b 740898e9-2cb0-4add-b566-85c1b91a27cd
80 1795180089.466711 warmup 201 0.0107269287109375 15cfdf5c30ad4ddf91b2335a7edf488a 0769beef-2b9d-47b6-9411-8398fd812edb
81 1795180089.7104251 warmup 201 0.020959854125976562 7948119f04dd486aabfa051f99212538 fd8234fc-0b33-4ab8-93e0-2d81297293f4
82 1795180089.963606 warmup 201 0.012027740478515625 742a4471437c49d18389b06e805a4ab3 b94f2545-4ab8-4adc-a3f0-3be6d2651cb0
83 1795180090.209379 warmup 201 0.0351109504699707 485f834821ee4901800a56167fc57a58 e092a5ac-578c-4e14-953e-e1a0f35307d0
84 1795180090.466652 warmup 201 0.011600017547607422 530e48d202294f02a057575c96939e09 a4d65f08-4d6e-433e-a26f-829b382a7af7
85 1795180090.713542 warmup 201 0.02402806282043457 eb8528092fe340249c57b787262eaa7b cf3c1f85-16f0-46b5-8967-ffb1ed355a34
86 1795180090.958018 warmup 201 0.031564950942993164 066321c9365c4ac389b0ccc0246b74ac ada76196-4fab-4482-83fa-b1f3b5eecb9c
87 1795180091.20841 warmup 201 0.047250986099243164 6e9c7ee6a8224292a5309c9efcfc6fdd c010493b-68c9-4411-a131-28f51549c637
88 1795180091.4665458 warmup 201 0.01369023323059082 4f35ff3427584bc599fffb38720c8844 a5a00939-38ad-4653-8c3b-eb2f6b6dbc4c
89 1795180091.712262 warmup 201 0.04745006561279297 9a10a79bd165470f9d80c1b1af2033e6 c24bfdf3-5d25-468e-9624-70f7dbcde2e6
90 1795180091.956745 warmup 201 0.014162063598632812 c720e30747a0488fa74b22944a2cf758 15343548-7567-4d6d-973d-5915cd560bd6
91 1795180092.2134972 warmup 201 0.041545867919921875 58a18bbe2e4c4fa3bbf977564dd3b070 c744740b-c846-461f-b148-dd573f0400df
92 1795180092.4640179 warmup 201 0.1579892635345459 5ceb3ba17b6f4b6fa82e83dd2b45a550 55f782d9-ba6e-4fb7-a0b1-2ef0b321a085
93 1795180092.7122822 warmup 201 0.018977642059326172 8cfe624196ad4a108d25effffb13bb76 37c97b83-8484-4fcb-9c0e-6cfab9469183
94 1795180092.957189 warmup 201 0.07108902931213379 17f94c7584e84f8998ead6d44f40e85b 9edc0874-26af-41e4-85a6-83f49cb22fc7
95 1795180093.2155 warmup 201 0.015893936157226562 4f50a2f75a004d2890d0444434b2c654 02165777-9f1c-4285-987f-eb71c5a1c4da
96 1795180093.466285 warmup 201 0.010670900344848633 1e70aae6212e4411bc77affcf76c9122 7ce7a83c-43e0-479e-920e-6c45ddd097c7
97 1795180093.716158 warmup 201 0.01917099952697754 e7f7d36ef7e34852a270e54064609d93 4edee8eb-99b6-48c0-96a5-f24d939999df
98 1795180093.9611259 warmup 201 0.033547163009643555 aa606a9422d84863bf1e7cce2e1e99aa ba8d2d08-9473-490f-9fa9-569c37363808
99 1795180094.21668 warmup 201 0.017546892166137695 88519bbc68b745baafe3f587e050ed12 34cc6507-b740-47e1-a9f8-12fb125c7168
100 1795180094.463606 warmup 201 0.02402782440185547 f2cb2077364c42eb99f97cc6e9cc411d c84a275e-a365-404a-88b7-7619802c0867
101 1795180094.7084239 warmup 201 0.06055402755737305 ae3a8b7725d94141a71f3a4bb7b6d3a6 b51d198f-c258-4c31-9fb8-699f4dd6a649
102 1795180094.963558 warmup 201 0.011219024658203125 cb338827715d447281ca1cfb7baa6192 b9eeefe0-18b8-4d41-9a56-7e1a264599ca
103 1795180095.2135942 warmup 201 0.017534971237182617 048f292244af4d128526d24968d99cce abe30323-cc5e-40f3-b0c5-d2fd04779543
104 1795180095.464899 warmup 201 0.02784109115600586 ca74025698904eb7a7c1b881dce81fd9 cc146ef6-43f7-46f5-b6b7-ad8f33422a17
105 1795180095.712756 warmup 201 0.06147313117980957 3b9a5cc7e14846f190112798ee495f8d 582f8ca7-ed3e-4635-bd14-3f086ae088e1
106 1795180095.961024 warmup 201 0.02721571922302246 3f9e74f4846747249dadb045a84612f1 b9164fdd-cfdb-40e8-9972-a7bb15331b96
107 1795180096.215352 warmup 201 0.011113882064819336 230ef86450f5409f90b0f3b72f13477b 426eb3eb-84ac-4871-ac6d-edfb3edfd9e7
108 1795180096.465764 warmup 201 0.010383844375610352 92977e8d8ca94f42b9f82b6760f37357 379c5abf-56ee-43d5-b7c6-9e32af7b3f83
109 1795180096.713002 warmup 201 0.010370016098022461 41ead6618f494679b7b1dea567965717 481f589f-948c-4447-99cc-fde77608cb8d
110 1795180096.9571269 warmup 201 0.013314008712768555 48a1a94c03674a668fe50472b6ca524c 0e64fda1-874a-49e1-9c0f-06a137d3a3fc
111 1795180097.207125 warmup 201 0.03571009635925293 0855de458df949c0960a19a3d6f81481 3c40f46b-c28a-4b51-9f7a-0e3747104429
112 1795180097.4657059 warmup 201 0.025941133499145508 fc09a297445c4255b337f8581fb73d0f 3b06592d-e56d-4006-96dc-13fcce9c40b7
113 1795180097.711473 warmup 201 0.019830942153930664 cf215ce3ae6841678c323fafc036c13f 9f445733-c50b-4690-a50c-900e3804ef95
114 1795180097.956442 warmup 201 0.011531829833984375 abba07cb6ab64b16a586f2193a8646c1 b931ebd0-fb63-4562-9417-7ac764ca1dea
115 1795180098.2144608 warmup 201 0.014567136764526367 e8d2e2539d774d88aaa2c4473977c67e df144bb7-3ed3-4e30-b65a-194064b53183
116 1795180098.4572449 warmup 201 0.08848404884338379 6bd7fadb57c9417fb94e1a3419ab8cc4 271ce552-30fc-4efd-8cca-4eba7419763f
117 1795180098.708143 warmup 201 0.0271909236907959 5afd4658539842928dacb103fcf9de04 cce0b416-63c7-4dd9-9493-f8fe4a860a10
118 1795180098.96469 warmup 201 0.08780908584594727 b62003bace334091a3b8aa60404be850 bba8ff6a-4a76-40e6-bd6a-281ec6a96868
119 1795180099.2078679 warmup 201 0.02692413330078125 9e650dbe0af2454eaa8998c57d0cf42b 481bd894-496b-4266-b6b2-0e44cc1426a3
120 1795180099.4654238 warmup 201 0.012391328811645508 8b69c1152c28444b8d903d040f63d724 d66a387c-8a71-484d-878c-c6039df6c7cd
121 1795180099.716196 warmup 201 0.013508796691894531 08a4e30d9cd040289e88f1f32e6a6cab 1b1ecdbe-9293-41d7-842a-873e3ae2c49d
122 1795180099.965012 warmup 201 0.03931689262390137 f81e66c40e4c40c0a3574da6f042285f 9c2c656f-1486-41ce-b120-0a55526c0d95
123 1795180100.2069771 warmup 201 0.027806758880615234 29c3c90fad8a4ed781afec70fc431013 018459b3-7916-44b2-82c0-c7390a0b7ac1
124 1795180100.460327 warmup 201 0.019770145416259766 a2d50f5e1f124fab88c3ff5019c37652 89eea49a-a704-4a3a-af3d-21309c592214
125 1795180100.713459 warmup 201 0.022834062576293945 dc517a51ea684bc993695fce6571b5aa 87ace08c-2856-4ef8-8482-1c81e698894d
126 1795180100.9633029 warmup 201 0.011710166931152344 ef190508073f479a9d57bdb4c9b9c1e9 64bec410-474f-4412-b34f-8aa8d7b9e3b4
127 1795180101.216318 warmup 201 0.01240229606628418 7ee886cbc5094be3881d8882c3dfe9a5 f5fa9bfe-6f92-4b28-a439-7937370a2ed5
128 1795180101.460207 warmup 201 0.011929035186767578 e1a6eb07742d48729e6d06a3cef34df7 3530d4ee-e604-4b01-8f12-7b1441f2d50b
129 1795180101.716116 warmup 201 0.010046958923339844 0c100de439214d3ca293b8917cf9c575 f725ce7f-93f2-4e97-be37-d41ef73982a9
130 1795180101.956589 warmup 201 0.01887202262878418 bd9178c605734da7bf53e692c7fa269b a085ee86-d766-4b74-8568-5d3e1d9ceec3
131 1795180102.2150679 warmup 201 0.04256629943847656 7ab9325f6b5c4364ab96c9369224cf1c 6835f67a-7cd0-4d11-ab98-e40d44693689
132 1795180102.462175 warmup 201 0.01769113540649414 8f392006bf984a59aedef30ee800156e 26aa8e16-eba7-4696-b32b-8dfb3cf13d74
133 1795180102.709732 warmup 201 0.04403996467590332 344073f83eb3454faf345c87ae4c89f6 333d76eb-d994-45bb-a987-c2c146bb8562
134 1795180102.9586382 warmup 201 0.015874624252319336 ce2c60ba55fa4829a3a477511b333ffa fd203964-ff78-425b-8c1e-881e0323ec05
135 1795180103.206547 warmup 201 0.013957977294921875 a577df3378ff4b5fae4c2e0f7bb8bd24 929b0cb8-2692-4995-8dcb-5c61cad260c3
136 1795180103.465699 warmup 201 0.06361103057861328 b1716a8aaa4d474eab1ea16c7ef710c9 f076ab04-d83b-49be-b026-73bf0c205d02
137 1795180103.7078888 warmup 201 0.011960029602050781 5bf8e68832564e3d8b0815af5d9f4326 6928bfbd-e29a-441f-a048-635a466c73f1
138 1795180103.95976 warmup 201 0.014333009719848633 d8bdd91a857745d6abe1ed3e19aea098 5e559263-dd52-4050-a13a-ae1441a79556
139 1795180104.210141 warmup 201 0.0119171142578125 ec5ed7bd28ba4cec925e85e563b22928 3170e5ec-95ff-4f0d-b8af-9b8803663b1b
140 1795180104.462445 warmup 201 0.013860940933227539 1cceb6665907453fa5bb35ae531e06ae 099f91a1-f991-4315-9816-1b8dac80fd90
141 1795180104.716085 warmup 201 0.027819156646728516 b569df314b2d4c28bdca00cc5638ec82 457f96f9-fe3d-4086-b0c8-bc1148502545
142 1795180104.9571462 warmup 201 0.010566949844360352 ee53243220954c65be8a449284fd22e8 148c93de-1a65-48b6-84f0-ae15b9be6d50
143 1795180105.2108119 warmup 201 0.01622605323791504 a8009e01d96f48dbac64193fd0fe557c 200a85ae-6619-4391-8f21-56ce2877b3da
144 1795180105.465821 warmup 201 0.01184701919555664 aa99ea688eab46af83e302c28ce7d756 2ab663cf-6898-4cc4-9e43-8e40ddf480f8
145 1795180105.7093928 warmup 201 0.051905155181884766 0aaee39c361546b3849fd152947d9cae 8272cf06-6e91-46a8-a1d8-9d08e65c331b
146 1795180105.9574258 warmup 201 0.016104936599731445 e862f37799a8410686508c3ed816d7ba 9de19f12-24b5-448d-949f-ad8adbc32650
147 1795180106.215368 warmup 201 0.007997751235961914 558cd74d71da4cfeb1e6785e4678b33e b8b88a48-1e5f-4f27-b46b-edf215c0e51f
148 1795180106.465971 warmup 201 0.016330957412719727 8ca7061c259749d890c2ae7a24ee10ea a0e5fd6c-1db4-4898-987c-16f37224951d
149 1795180106.7134972 warmup 201 0.01929187774658203 96db26073ac04a9c8483d94d36d648c5 ba3b1aaf-9966-443c-8bad-8f3f5cbe772a
150 1795180106.965224 warmup 201 0.016424894332885742 49c438ff3cc34374a64df5a646ca0064 c04e7e61-6934-4b15-9f27-6992af1e740c
151 1795180107.2087429 warmup 201 0.13824105262756348 91a3a6e513ef48e091e5fec6cadc2651 c15ae37c-a0ca-4002-94ab-63c613b8a3f5
152 1795180107.465683 warmup 201 0.06748700141906738 a0b0fadee95b42d9846c6bc19f6a3425 fe21da70-979c-4f25-bd7b-ecec462a4f6c
153 1795180107.716597 warmup 201 0.043663978576660156 d0a18a5e9d3844609e1846de52fdc334 09b03827-9758-4c19-96e9-02245b95408a
154 1795180107.9577112 warmup 201 0.05071878433227539 8bc17d2ac861457d906edf6bb733aaa6 d40b8a9d-314e-4e6f-a235-53179160e38c
155 1795180108.215978 warmup 201 0.016988039016723633 b9d90c9fef9b4cd2b40f0d01f960a800 c2cad002-8df9-440b-8d06-8e9edb144d2e
156 1795180108.456348 warmup 201 0.013724088668823242 dd7f64e94a8e49808c688bc6e7705f9c 49bf1186-56cd-4c5f-962f-c1cce4115bf9
157 1795180108.716038 warmup 201 0.014075040817260742 46027f62899f44aeb5b997155d9dacd8 7800d167-6a9d-4299-b585-a8e422b6ed15
158 1795180108.9656162 warmup 201 0.029653549194335938 c0328e86f791432aacbd1351a7778276 c10794fb-2fbe-4254-b6d1-68c662d03d3a
159 1795180109.2134242 warmup 201 0.011368989944458008 78d77602d48648b9bb473b838ba80d94 0ffa1fa7-ca44-40ef-906b-91d7e6adac7b
160 1795180109.466222 warmup 201 0.011855840682983398 5a4539f748ee44139ed31d0523512e4a 03ed8cf0-6572-4420-a46f-f31c353d5ca7
161 1795180109.711676 warmup 201 0.016754150390625 ada1f3a756354017b1b3606e73a5e568 530d5f4e-8a90-400c-bd80-49c3282c6409
162 1795180109.957644 warmup 201 0.011628150939941406 069abe6e4e40491499fb9c0e060b1f02 fd454afa-45a6-48a1-acbc-a30f86ea1611
163 1795180110.209734 warmup 201 0.01928114891052246 feac37a20259432c87e036d5a3736f05 6c0e6316-6d64-45ca-82c8-a45d07998b22
164 1795180110.4659588 warmup 201 0.022558212280273438 83931cf8c4304d13bbbcb09fcbd09a2c fe5bbefc-b090-4206-940e-53fa504766b7
165 1795180110.7111838 warmup 201 0.026392221450805664 c86accc4673a4021b5846f199e507dc0 aaad07ac-8808-4d6a-96df-28b4fe06feba
166 1795180110.9606888 warmup 201 0.02469611167907715 f2617744f0374aca828bd5e16a522317 a020e190-4f8b-4ffe-9ba6-454b67dab903
167 1795180111.206548 warmup 201 0.025948047637939453 d08ad39890994932b23b2e165e0cab6a 26fa8160-8bf3-4577-be99-c151ae1f91f4
168 1795180111.464929 warmup 201 0.021355867385864258 4ff40371a3fb4209a7995e9993296026 6f20343f-07e3-4fb3-8413-55ca6218b45f
169 1795180111.706388 warmup 201 0.011980056762695312 1ba5d021fa704d43a494e01106455fff 889d6919-d410-4fb9-8e6b-d0313916c9c4
170 1795180111.965568 warmup 201 0.021121978759765625 9026ed4c64eb4cfe9d358a97604851e7 4022af74-df48-47e2-a3a5-ce4c4eb29cd8
171 1795180112.2073479 warmup 201 0.016837120056152344 c02d63b9da53468cab2945d158c88217 43c2697b-fa58-47cd-8aa8-0017553f1a0f
172 1795180112.463627 warmup 201 0.009462833404541016 0b96e9c8026b4acfa6803e63f5d78270 61b8558d-3fbb-4fb2-a19e-143e173783b8
173 1795180112.713996 warmup 201 0.015658140182495117 bc6d171122fc4a82831317d5081c8c71 2564d01e-c563-4c1b-8d48-9f4c7625f03d
174 1795180112.959977 warmup 201 0.03197908401489258 26c142b3d1f346afa03037a9fd03f264 f21f69d9-3855-492b-8e12-c1c3173ff82e
175 1795180113.461932 warmup 201 0.3604569435119629 e0de43a93b9747ed8135916b9161edb8 88a841fe-a14d-442d-96ca-f9ccbbd90a85
176 1795180113.963574 warmup 201 0.0680840015411377 7f5ddf5140ca41eca9f99005e3be922a 1d915906-50a8-4e50-ad96-d955eae5bba0
177 1795180113.2087371 warmup 201 0.8404510021209717 266ef7a83b1f4ac6943f15629354bc1d 46a8f038-4ecf-43c8-af7b-a76681b0b7ac
178 1795180113.714091 warmup 201 0.3419630527496338 e3408be2e18243fe952dffc3a29d9f76 6bca83aa-886a-4e16-87ff-c4cbe343b04b
179 1795180114.21387 warmup 201 0.04325413703918457 531bfc2b57cb4a4abd2bace25b1b206a 56501d24-4338-4a51-ae9c-d571359a5a3b
180 1795180114.459767 warmup 201 0.07159805297851562 704037cba11f4f87954b914ca2dea4d7 9b7e9ffa-1018-4469-95ac-ae45816e1db3
181 1795180114.713313 warmup 201 0.013693809509277344 a8aea998a4a14b068536acb95950bd6f ea6691a4-cd8b-4aba-9200-27596e23d8f2
182 1795180114.9657989 warmup 201 0.019134998321533203 f16be51009bf4267afffe901c6f9ec25 15bd426d-2b24-433f-91a9-b50b78a05afd
183 1795180115.210476 warmup 201 0.015871047973632812 8a59d5636bd849c195436327cf03d6bb 7a92fb63-0ea7-4b97-97e7-627217b57b48
184 1795180115.4618099 warmup 201 0.017672300338745117 8d0924d1e80648d99ad4b02f3743e748 ae81b2fc-f535-4164-9d0a-849757f3de7e
185 1795180115.7133892 warmup 201 0.23829078674316406 ee868fc316c44711899a2a7eac1f983f d4d6bf84-798f-4719-8ce6-7b4f71e3edea
186 1795180115.963288 warmup 201 0.06793808937072754 be960e7d72fb41d686edb4c1519cd34e a4a8dbb8-dd44-424b-8d44-94338319fc2e
187 1795180116.216624 warmup 201 0.011794090270996094 f1becd1850aa459188902bbfe85dc317 195060c9-3ab9-4947-aa5c-90fc6e708878
188 1795180116.465261 warmup 201 0.010736942291259766 ccfe2471b6094bdc98d8f7c7cc5215c9 8598b485-22cf-494f-8e7e-093045e91e0a
189 1795180116.7087119 warmup 201 0.019225120544433594 6490a5b10731467fb70b893217fbe7a5 fdde1515-647f-46ed-be62-61a32f012d97
190 1795180116.957767 warmup 201 0.012019872665405273 c0f622e8a99846f1897ce6d9e4d24109 2efc878d-0126-4ee5-99c4-b75f1b08209d
191 1795180117.2127738 warmup 201 0.023031234741210938 245477e2a0714376b674cd53912ff0cc 7396abb8-7d06-47be-bf35-caeb9104a6b1
192 1795180117.465743 warmup 201 0.03672194480895996 b699aef6a81a4583ace9caf63f684ad9 5b27c6e2-7533-423b-ad6a-6ffcef894efe
193 1795180117.7158148 warmup 201 0.01888728141784668 162f7b95191d40f59651888ee71f53a7 72779180-5849-4054-8d1c-e9f50e31dd53
194 1795180117.96484 warmup 201 0.04443693161010742 41e9e30f1cae4d4bad5e3e51fb7fe6e8 a9de2cb7-13f6-4d4e-8418-8394733c75ef
195 1795180118.2133079 warmup 201 0.023710966110229492 3698080dd9de4cb4a3b9087e6f736e52 00a5f651-45e7-4306-90e3-034890c0810b
196 1795180118.460449 warmup 201 0.029680967330932617 ff85e90c255c47d0a8648258d71bef3e cfc32a1c-8eef-42bd-bbb5-7cf95dc1971e
197 1795180118.7111292 warmup 201 0.02736687660217285 cc2f0f325db64165b0e63db66ef2e912 946f661d-727d-420c-b2c1-10ccc5721ccb
198 1795180118.964362 warmup 201 0.014344930648803711 cd431bb740694c60a71b0c05f8af6974 0ffe6ffb-c402-4b6a-87c3-9bb8094ff16f
199 1795180119.211197 warmup 201 0.013164043426513672 d57df0e48f9848a0bcce0b113bd85726 4bf5b6f4-b8dc-4337-91e8-06d9dfb75af7
200 1795180119.4637692 warmup 201 0.011598825454711914 86002fec5caa4665b801fcf8dcb2e433 f863f2b4-5df2-4195-b2c8-9b153c392a53
201 1795180119.714405 warmup 201 0.024904966354370117 75aea0e8c93440a0b64f74afd2bab8e6 29929580-e108-4bf9-abd6-845383375440
202 1795180119.9656231 warmup 201 0.024814844131469727 f552643138364b199afa14b6418efd65 963ac84e-6238-44d6-90a7-959c7727efcf
203 1795180120.209861 warmup 201 0.019379138946533203 301dbfe0ea9742d8a6e972e148164e86 7b40e23c-22a1-4645-87d7-2cd7ecdab5e0
204 1795180120.460503 warmup 201 0.012269973754882812 56d19575ad6b4a159d29b2f28035a292 ce004c67-86e3-4611-b2d3-410c60eb29e1
205 1795180120.707459 warmup 201 0.016863107681274414 eb32c8837103450590381b57d9eb8a29 d4ac9a5b-353b-4cda-a097-06c588a077fe
206 1795180120.956764 warmup 201 0.03303384780883789 1a549a26298f4650b8ff2effb4231aa3 304dae29-1b9f-45c3-8f80-c3ee5ab8291f
207 1795180121.206913 warmup 201 0.01592087745666504 7d5af7fc0b024ea5825a6ed2a2638e50 5e90cd2f-4070-4810-a1a3-e27604dcf771
208 1795180121.456815 warmup 201 0.011278867721557617 76f880ca36c241d888ee2524fc8b2b3d 8d9e023a-99d1-43ca-ab5f-3dd6d9a5df32
209 1795180121.71327 warmup 201 0.025320053100585938 3df813a8d9c443ffbb44b524bbb916e8 28c64eb0-cc2e-4f71-8ede-8e852d15c2c2
210 1795180121.95684 warmup 201 0.00838017463684082 5531422685b943a6a369e608c058fc57 591f8ae2-343b-4273-a2f8-b6cf48646f32
211 1795180122.206609 warmup 201 0.00858306884765625 bb7288aba8434bba9fc41666e0904fc1 24f71b02-74a2-4901-8200-9cd545dcd2c3
212 1795180122.462899 warmup 201 0.015925884246826172 69d4f4e4b7544c409204aec3d83da9e7 e74f7431-65b2-4e35-853e-cec86896c78e
213 1795180122.709456 warmup 201 0.010297060012817383 7ae458c18ec5447c8ac6c6118bc18fcc a090d8ab-702a-4b3f-a0ea-d4e383b45cd2
214 1795180122.964902 warmup 201 0.013642311096191406 a7a71c5177a6460a8c2b7a8a5127a8a5 bf2acb88-b590-419b-8578-3201c3381a73
215 1795180123.2154398 warmup 201 0.012851238250732422 b41e6dd0417040dfa5ef0bd7a51d4d5b 4486c43c-b469-4899-9d82-2d6afb8f03c3
216 1795180123.455821 warmup 201 0.013101816177368164 0e0788c63dd3458a988dc582fc97093e 0914e390-c13f-46b3-bcdf-998b9deac79b
217 1795180123.708153 warmup 201 0.020342111587524414 8e3ad121c9184193b84eec5c7700ff37 aa11cd4a-7649-47d7-ad4a-4daba45c64a9
218 1795180123.955742 warmup 201 0.05740213394165039 c1a8efda79ec44e0a3c108a65ecfe280 70604d3a-50e9-4c70-9337-04b65fdc4809
219 1795180124.208821 warmup 201 0.016861915588378906 cd2340f1dfd248ebb7d5356f79f8f810 561a47d7-b772-43a0-9033-1868fa5ac4aa
220 1795180124.459167 warmup 201 0.08424186706542969 ce5f4ebaa39a48cc9925177ddd46b951 60f54ad3-215d-43a1-8118-6c72a5efa4c3
221 1795180124.7120728 warmup 201 0.0500941276550293 12adf6cb9cfa46e4b59d0d827ac86be7 f523c477-67a0-40e4-822b-bd6cda069068
222 1795180124.955792 warmup 201 0.46666622161865234 cde8f775be3e45d09e4f031ba0836095 ac5ed17e-b55c-497d-a88a-93d5612ac03e
223 1795180125.2137432 warmup 201 0.21810579299926758 7789893b847749a39412cb64dad122d0 c7c84df7-5b9a-4f26-a31e-ebe6ca26d14e
224 1795180125.4653718 warmup 201 0.4004700183868408 4c692c0c992e4cb4983d702cb7f2906d ab70beeb-6847-4079-a880-272308582b3d
225 1795180125.7116349 warmup 201 0.2136549949645996 4a952313b6c241e7b24648f925c91c87 396d5aa0-9d6d-4f66-b11c-47654ba5060a
226 1795180126.20828 warmup 201 0.022182941436767578 4b2f49632f6f4378a862a76bf2b0a11a 287778ad-cbcf-405c-9781-76c32785d4f8
227 1795180125.957431 warmup 201 0.4076690673828125 c19fbf5dc41f44678bf65a75718f4763 ead5c9c8-736b-4cca-a826-797020bc4952
228 1795180126.465112 warmup 201 0.11259102821350098 35523a266edf48bd943263384c9787ad 70c1baa4-3067-492c-97e8-b70ddfd400eb
229 1795180126.70634 warmup 201 0.1282970905303955 08db262afecd475faffb5c5151eaec0b 5a306129-f313-4586-afc8-6ab0d70302af
230 1795180126.956701 warmup 201 0.08037590980529785 8b66a1e2b3fe4bc5aa66698dc7534c95 58c38c8b-047c-43ee-8e2e-a105acc3960f
231 1795180127.213266 warmup 201 0.0364232063293457 023627bd9b8f45bc8335c9d02ee6a0e2 c209d4bf-212f-4267-b115-8456a3b6cd8f
232 1795180127.460497 warmup 201 0.018844127655029297 d66c88144827455095a11c5bb3a8ab75 74a961b6-8a03-4592-8477-8c79765f6135
233 1795180127.70784 warmup 201 0.013520002365112305 a5dec999bcac4e60ae5bc69113c4b97c 0694641f-063a-47ab-ba69-446a24b0302d
234 1795180127.9577322 warmup 201 0.014217853546142578 2968ad48558e4ed5a5a282176ce12a28 211849a6-68f0-4974-a009-8f3056aac6b0
235 1795180128.2072031 warmup 201 0.012610673904418945 8e65ac12d21243be946656474b9be0a9 e612a563-7d2f-439f-a64e-6ad7902b758b
236 1795180128.460282 warmup 201 0.023057937622070312 66f10fbc943e4496a3ff400a3e46ed0a eedc2fc5-c6e8-4613-88a6-77a6637c4584
237 1795180128.713231 warmup 201 0.03037095069885254 31fd1195cb3847d08974df206fc6b0d7 c45a7609-ad92-437f-889c-ce7962d075d8
238 1795180128.956662 warmup 201 0.028537273406982422 9e68be6d44d64037bf3006185c2d5d23 4d731559-f9ce-40c0-95ef-8633fadcdf99
239 1795180129.211051 warmup 201 0.0380401611328125 03b2f65668d34f15b6c9546be47fec86 94b225fe-050b-4be4-b9bc-d85c7180abd1
240 1795180129.463252 warmup 201 0.03919506072998047 9d7008be8e4b4e6b84dda2204dd3a0f7 ca4fc962-d814-42f1-81ef-5ac07d99f4e7
241 1795180129.705261 warmup 201 0.01246190071105957 5b7a9e05ef18435caae5c26fd0fbba55 c780199c-588a-4089-8e1a-b9e29040dc94
242 1795180129.958072 warmup 201 0.03585696220397949 80f818ad28014402851cf0dc4fe6b6fd 70fe8691-9a92-40aa-aba3-d66c7eaeb224
243 1795180130.2058141 warmup 201 0.09956502914428711 29d9e9488c834b89b5811683ef978cdf 72350268-eb1d-48f7-9c64-20b973f464b5
244 1795180130.465339 warmup 201 0.025896072387695312 d7b5a37e9f774ba09b755dc347a6a543 2abf41cc-ed1d-42fb-bd0b-c4dd09ac95a1
245 1795180130.7066429 warmup 201 0.011258125305175781 138cf5cf556a46bd8c5996af9771a756 fcf61d52-05f2-40f8-b719-028a7d3799cb
246 1795180130.9631832 warmup 201 0.013113737106323242 6cd0c7e99e144c189130e48bd586ebc2 e384af86-879a-49eb-b88d-27fa49ae5387
247 1795180131.2116659 warmup 201 0.03482413291931152 4bcea84186e24a79a32ea26db27b9110 652c07d3-b5a2-4b5a-9739-34d2340bbedc
248 1795180131.4630752 warmup 201 0.01401972770690918 dd7c079671544c79848adf21e81287b3 04f4b354-d35e-44c6-9cdd-5c0701c50001
249 1795180131.707235 warmup 201 0.01443791389465332 7f24ba99e54449eaa6dd61923148169e 2620aa56-3e44-43fe-9f07-9608321b14d6
250 1795180131.9631941 warmup 201 0.010252714157104492 5bba090e4ec542649013171deb048bbf 5cb08e95-e5a4-4ab9-bd2e-57c5f7024413
251 1795180132.213757 warmup 201 0.01481318473815918 e7d645f2882348cb9e340b8993c8bcdc b82f4159-b720-44f7-bf06-8d5d91c7733a
252 1795180132.463203 warmup 201 0.024817943572998047 352c5922524b47a386900aa4e676ee34 6c286b53-7960-4d83-9885-04ac3de75eb0
253 1795180132.7094731 warmup 201 0.008898019790649414 f4035257b55c41d5921cfecea8966176 574fb6a6-2aeb-465c-831b-a64ea30727c8
254 1795180132.956722 warmup 201 0.010103940963745117 afd60764b6934ebd8a725d78bd3f141b 7972d992-f8a3-45d5-9bb5-4314a6553f51
255 1795180133.206574 warmup 201 0.015628814697265625 1721768d2aa64664ad3e327db144300a 1edafeaf-e173-41b6-8c3c-83f18a8166c1
256 1795180133.456523 warmup 201 0.011080026626586914 f15339e1ec4d4a6ca8e331011f988549 f72cc398-eaf7-4d27-8b08-c301779f1dfa
257 1795180133.71329 warmup 201 0.013360977172851562 c96a9745a1b74d58b3dab2cd5645c4a4 4ac57986-d879-420a-b1e1-244e561e2452
258 1795180133.96325 warmup 201 0.0173952579498291 6f2a1136ff1042d58b8f6cd077189097 cc1000c0-ad46-4409-a37d-6ff7df3275f2
259 1795180134.2074082 warmup 201 0.013551712036132812 1c33f13207324f7391e00ac92f57a94e d2e5b4fb-af9e-4279-8584-8bf2c0d76a29
260 1795180134.4623308 warmup 201 0.007900238037109375 34e1fdd2284a4dc388d43c3da2725df8 880018cf-c8b4-4395-b5f5-fa1c8fa65273
261 1795180134.7132852 warmup 201 0.011044740676879883 6403402a1ce84f2f8806a914dd8215b5 61f00d79-ccb5-494e-bd71-d862ffeef5d1
262 1795180134.9608378 warmup 201 0.02025604248046875 0894586e34c54ab697c597b7421d034f 3244ed3b-f15d-472b-9e3c-8d3e924f5ff6
263 1795180135.207734 warmup 201 0.011043071746826172 1b06517ab2a444179728fb8dbfca0827 03d2f9ae-3ebe-4dee-a81c-6b6c406f9d2e
264 1795180135.455019 warmup 201 0.013014078140258789 fd6c4f6cf4ef4e1db2ecda69d20735cf 89bd0345-ec51-46d8-a006-ff0a2d410b5e
265 1795180135.712738 warmup 201 0.10438084602355957 8f09b5249863497e94b4ac5dd86ba4a0 c23262a9-cc13-49bd-8f23-795dccc65e49
266 1795180135.9571142 warmup 201 0.018002748489379883 f075fb7c0d434b67a026427a792aeec0 351ccb0e-cc60-45bb-ab0c-8ec9d6340b0f
267 1795180136.205044 warmup 201 0.011342048645019531 fc8eb864844d49689747e262be617164 f250da35-d8a5-4b94-af66-183bb0c14d81
268 1795180136.463886 warmup 201 0.022532939910888672 1e2cf1399e5947a8a3d01c53e008b506 346b4b23-4597-4a11-8206-cd77d95da8cb
269 1795180136.7051678 warmup 201 0.0228271484375 1e97029cac1f49d3936d82698cf0f18d 98396eee-0cfe-4362-b38f-479c9d56121a
270 1795180136.95592 warmup 201 0.01760697364807129 fa15a0606f634b378faef6d7851b21ac 5c6bad4a-710f-4f13-b688-1c694c8ba23f
271 1795180137.2061172 warmup 201 0.012992620468139648 ec2ffdb4bad8408f8203f6c45fbd2bfa 722c2c2d-e8df-4960-ad16-ba0705f57818
272 1795180137.4616551 warmup 201 0.011270761489868164 e03220cc975a40b2866f11bec428e421 18f123db-6131-4d5c-bce5-8cb6857af849
273 1795180137.7097511 warmup 201 0.01858997344970703 e38529e88ca9466aad28b852ee566d05 d6fb2504-540c-4455-8431-c5a800f7536b
274 1795180137.956691 warmup 201 0.015127897262573242 7ae2111789d54a03992356796a309d08 1f023142-499c-431a-a878-710b377794c9
275 1795180138.2149088 warmup 201 0.020531177520751953 5775e215d1484f9ebbb21fd79036e3b9 5f8dc182-154f-4d6e-8720-6475c7823c52
276 1795180138.45882 warmup 201 0.015606880187988281 e0f4da4791f041d0a1b626e48732b09a 8349e524-f496-4d09-bc55-b25ab82ac628
277 1795180138.7100549 warmup 201 0.012995004653930664 c3461393466d4972b02dfb9c01957188 5bde5b66-1212-4260-a66c-5f7657e5f10c
278 1795180138.9580622 warmup 201 0.01614689826965332 57c5d3fbe0be4fc49cb0fdad8aadfa92 cdf0eecc-6a30-4541-b9f5-ddef849546fb
279 1795180139.2118132 warmup 201 0.013647794723510742 767e969ca9014b6c9996531ce559df61 88609695-abdd-4bad-96db-9fe814702590
280 1795180139.4632142 warmup 201 0.009091854095458984 f86e3591b7e64a71b5553f371b7f40c9 de93cccc-76d6-493a-ade7-3ea0a0d5fb71
281 1795180139.714897 warmup 201 0.015331029891967773 cddf319c99ef408c82bfb4d343ef2635 8fa78445-3d8c-45a7-96ae-01a941f020f0
282 1795180139.9627159 warmup 201 0.01434016227722168 3fcf4fb003844ec8bb54ea1f5f733efa 6b4ac399-f00b-4119-ab42-0af7df3f357a
283 1795180140.21356 warmup 201 0.049919843673706055 78da2033cd6746b3aacecb1a52a94bff 43f75c5a-7986-4c87-b2be-643701e8a30b
284 1795180140.463584 warmup 201 0.013402223587036133 e9561b1d921a4901a0d76129b05447a6 0bd74017-3e94-4f66-8019-b964128d33be
285 1795180140.713254 warmup 201 0.016618013381958008 c8478479d45f43b6abf82abeb4fa83ab 1f35207e-990e-4b2b-9b12-8b2ca300e495
286 1795180140.962937 warmup 201 0.019067764282226562 51f2dcc6fa61439d9b90a247c2f15144 1a4ee6e5-a8aa-4b57-ac2c-f6d7a8605e02
287 1795180141.214133 warmup 201 0.011148929595947266 acbb62edba674fdbaf5c001a80efa143 1bed9e98-06d0-44aa-99ef-7a0ba428d72b
288 1795180141.462243 warmup 201 0.017611026763916016 a80c5bdac3b848ac89ab8d1132f29295 81969480-34ca-46ce-8724-090271f2d3ea
289 1795180141.713085 warmup 201 0.02792811393737793 4a1c8d5e101e457aa1bb3532c62715ff 2a4755eb-58d0-4e0a-8f04-c7e085bb720a
290 1795180141.963087 warmup 201 0.01384878158569336 adc2b0e3598945b4b1b6831405aea2f2 4437fa0d-be96-41dc-b191-332f69d6aadf
291 1795180142.204875 warmup 201 0.01084589958190918 09891c3bb41d4df9836d56c0ad0e942f bca6e934-63af-4bb9-956f-e53c7205efd4
292 1795180142.464761 warmup 201 0.010806083679199219 660ab3f0b0ff49debf10a44783abe0fc b83355bd-9cb9-47cc-8a37-295e91755f76
293 1795180142.71471 warmup 201 0.00988006591796875 3ba7601657da46a588e8a6e9ef71cd8b 20c7f9a3-b284-4af0-a23b-c16479768a75
294 1795180142.959651 warmup 201 0.015625953674316406 eb71302e239744f2a759cd4f4902a058 1c3ae2d0-920a-45e2-8914-0e71924f5a05
295 1795180143.213386 warmup 201 0.009329795837402344 ca97cb82e52544f6a5c4400f7bb5f3f5 bea54320-8738-46fb-9f20-0ebc92a9d599
296 1795180143.463989 warmup 201 0.013462066650390625 9ac1e64e71e84d41963aceb41d3c9331 959cec4c-2063-4e65-ac56-29b3733739a5
297 1795180143.707256 warmup 201 0.0188601016998291 d18b958f46c247569415bfab8c827026 5acac01c-4e9c-41c0-8153-81cfb3f4879c
298 1795180143.961851 warmup 201 0.01336812973022461 2f1a398627144fbdaf3f2e478d33b28d e3962383-7d5a-4a92-877f-249f3bfb0f41
299 1795180144.2063081 warmup 201 0.011801719665527344 39a64faf76494e99a03a888cc7e2e4c5 042bd8b2-5e39-448f-8fb4-0f5278a920c2
300 1795180144.463094 warmup 201 0.012636899948120117 db742f1e32d6433fbdeb0ddd3da29f97 f5fd8b85-e55c-4e8a-8870-1d645bb46a20
301 1795180144.7092211 warmup 201 0.023552656173706055 64f7b49a036248d1a70980d117a0264c 019eb5a9-db89-4439-b044-849894838ff4
302 1795180144.960939 warmup 201 0.01747727394104004 a2f1fee7c68847e5ba5d2cb6e950c8c9 6992a0cd-0f41-4045-99be-c229b8450d68
303 1795180145.206008 warmup 201 0.02232813835144043 e5be693bf9654a7993d12aa16398e524 8061e634-e325-4d3e-9c3f-79f13edb06b8
304 1795180145.464622 warmup 201 0.025964021682739258 89689b110ecd442f8d823fd5ad1ef546 d225696c-a1ab-4f82-b65b-0ab6f411b87c
305 1795180145.714587 warmup 201 0.013881921768188477 e5b573ab906d4d47bc283bad0c283bde d0cd8d44-e388-44dc-9939-31448705398b
306 1795180145.956403 warmup 201 0.017344951629638672 1b12f10f44a54f87a2f82e80da2ca146 1d61c6a1-87c4-4622-9f7d-a1c5b97f3f1e
307 1795180146.205685 warmup 201 0.01906418800354004 be5614f0646e4c6c927704b32c216845 9ba1718c-e5de-4c71-a971-e1a06705f187
308 1795180146.46311 warmup 201 0.023369789123535156 070ea00c630b49c39cd773bd022a2440 abcec0a3-3c91-4598-838e-9e4b098828e4
309 1795180146.7096572 warmup 201 0.11615276336669922 134e4f4d39724ce2b453908df6ee3c32 6c9588b6-3e39-4335-ab45-76980ca22058
310 1795180146.962498 warmup 201 0.015411138534545898 bc00ee055e934794aedf46848b6d9d4b c293e145-6d05-4dd7-82c6-2c08595ce25f
311 1795180147.213019 warmup 201 0.01767706871032715 211a0e09715649478212fd645fe9c512 1733a12e-bb1c-4f91-a5d5-fb7d8e5254ca
312 1795180147.461827 warmup 201 0.08181881904602051 776ab497422a45e5947ddb531dc82433 8703b6e6-3869-4195-b083-a9bea68a7d37
313 1795180147.704898 warmup 201 0.010509014129638672 7c0b031447754345987cecade08e1e66 68a41089-abbe-463b-a62c-86014f741453
314 1795180147.955076 warmup 201 0.013746023178100586 533292e1cfc5425c80060b0f062c7e74 04cbf080-c35e-4ed5-89a1-02a9696d51a3
315 1795180148.205858 warmup 201 0.025356054306030273 1b139f377f83459caccc028d4b2911c8 77c18782-d7e9-4c28-951e-cc9f4bac2a8b
316 1795180148.4552372 warmup 201 0.01289987564086914 b98cae53f29045d19a8f133e79c9dac9 a0036146-0213-4c84-b15b-d5d8779684b9
317 1795180148.706578 warmup 201 0.02797412872314453 904ee8f5c1d9496bb40cf6d3fa23c656 73824605-99c2-4c59-8b63-82dc61294f03
318 1795180148.963203 warmup 201 0.011016130447387695 5543a1f12ea64c4c9c45d2551f6f8fc5 8e3b04ff-3dfe-411f-894d-5986c58bc61f
319 1795180149.2050078 warmup 201 0.01116323471069336 72ac534ce025497f8d8d1528baf61292 bfdc61c8-accc-4673-a0b7-4523936a42a9
320 1795180149.461789 warmup 201 0.02168107032775879 f42dc751108b4816a1a2776541958326 15b9d117-ce4b-4523-ad0e-2efad6034a24
321 1795180149.710597 warmup 201 0.01814889907836914 d3247bcd8b974042a193c4749779bcc0 509ae033-f080-4c6c-9c22-e26b1ab8c4f9
322 1795180149.9608488 warmup 201 0.015266180038452148 947d80b1b7f24134a13749b0f9054e7c 51a092bb-8aad-4f54-923b-8aeba78e430f
323 1795180150.213044 warmup 201 0.030508041381835938 c2df4f426f204f9cb57c75398cfcd2d6 70a7a0d7-e40e-4f4c-ace2-b9244ddc88e7
324 1795180150.459798 warmup 201 0.015178918838500977 defd7634ee5749bb9cb56938fffb6354 19f848ec-e61a-49ca-a9ab-ee159c377d14
325 1795180150.714624 warmup 201 0.01801013946533203 3b9ebbbbbc004bbbb1173a1decbccb00 68500f45-3a93-4b17-a678-82f4250ebb0c
326 1795180150.964399 warmup 201 0.014775753021240234 d7fdec7d34c5496e96d25c4e8b6f51a2 921a3c63-3103-4882-8383-1e4950642aa4
327 1795180151.214231 warmup 201 0.015651941299438477 a919ca5a526b4cdd91873a89b3249d7f c75eb90d-4cd7-4419-a1ff-88b6634c5666
328 1795180151.456893 warmup 201 0.01569509506225586 7e65e6891eb749aca2e5cca772c2be51 3528cb80-0103-4249-9063-702d10cbd573
329 1795180151.712977 warmup 201 0.011497974395751953 5f9b3149a64640b79cb47627cfef6355 2d8814ca-6d6d-483a-85ef-5959661861a0
330 1795180151.9644082 warmup 201 0.009419918060302734 c8f9dd041f14439b8e0a88f48c057989 cdf1a975-bc52-4014-b534-59148bf6c73a
331 1795180152.2131011 warmup 201 0.014716863632202148 68761e2b15b64610b364c2a964c6d5a8 3c1cd970-d8f6-4049-9a7e-a574670f1920
332 1795180152.4601622 warmup 201 0.06042981147766113 0ce081bdcc584db8804a460edb6657c1 c0d66583-f650-4120-8f62-3d2c7ae54dfc
333 1795180152.706217 warmup 201 0.020451068878173828 a664c5f673114604aa95b08aa1b7e972 893f49c8-3afa-4766-b58e-c54efa12286a
334 1795180152.9611251 warmup 201 0.025645732879638672 569d9881d33b4e0ebcdb3b55833bdce9 b25ab34a-e7a2-42e7-8674-de4ae680db50
335 1795180153.204326 warmup 201 0.023772239685058594 d15bd802b80a4cf0b5c5b66fe0247211 ba2185ce-cfb1-480b-909e-a9776122877b
336 1795180153.462528 warmup 201 0.04242205619812012 fb3345a7c61d4504b216f861da365b7e 8ffba94b-38cd-4380-a563-8f19403b00b8
337 1795180153.7144508 warmup 201 0.024564027786254883 41c45e295c094077b016ed6bae9a2022 a132391c-bbb2-46b2-b424-bd6a93139ef3
338 1795180153.961816 warmup 201 0.03986001014709473 170e1f3a76bb41fdba945e2e71f6696a 30fba817-b082-4d0e-8ef7-752c23ee4af8
339 1795180154.20493 warmup 201 0.03506898880004883 5bf03ad7556f43718c11e5aa04c57eef a590d9ea-3b17-4f7e-bd1e-a634d5905445
340 1795180155.218189 warmup 201 0.7187919616699219 2d064afc92574e9dbabc65af8987a7d7 abff9a4d-707b-4626-96c5-698361062611
341 1795180154.4665148 warmup 201 1.4753811359405518 670d7efd7b35456ba9b976db307daa78 1a183e3a-1b15-4d3d-94e1-74c22a1d02fc
342 1795180155.711974 warmup 201 0.2379162311553955 5bb3401f86f246dd8ee36d2e60e31858 1d37a14b-850d-469d-8b2f-8cf48b9c1c24
343 1795180155.0022368 warmup 201 0.965947151184082 ae126ae644014ec5b1936e416ed2a435 e52afb5e-8df8-4ed8-912e-d83a28d52c9e
344 1795180154.709724 warmup 201 1.354740858078003 e77df7dd28f84c3d95a7cc15d5e98456 d8df3654-4ccd-4a1a-a4fe-d747ed6bb174
345 1795180155.473359 warmup 201 0.5913128852844238 fa2ad4a0d89f44c58b539416eb37b3b8 01cce25d-695e-455a-9b89-3277459c6471
346 1795180155.963171 warmup 201 0.10228514671325684 a90640776b8c48d38e0ec681f22ac39a 0301e0fa-5b43-4ec3-bde6-339af08a4e2a
347 1795180156.214087 warmup 201 0.018687963485717773 7f967f771f324b2da7abe59c4daef1dc 1beda72b-b70d-41ba-990c-e5a062595310
348 1795180156.462667 warmup 201 0.018568992614746094 737f8ea6e6d84bd3b51cffe92b59a69c 53dfeaeb-b8bf-44b0-b526-5b16bff819e9
349 1795180156.7076569 warmup 201 0.02150106430053711 759761e721b142b9a418803b6df9c027 a06894db-6c95-46d0-8e17-60b650716376
350 1795180156.963069 warmup 201 0.020072221755981445 73a497e1f8cb490fabd21547c58e94c4 b008b068-3ff8-4bcc-8baa-b2aa568e0628
351 1795180157.210342 warmup 201 0.02046990394592285 8779184ae9ba4e23a3fc2a99f6980866 a15fd300-6132-4d3e-b1de-7264c7edb3f6
352 1795180157.4589682 warmup 201 0.032640933990478516 60829f4073814d08a8aebc311277ff33 82e61037-01df-4764-ac06-21e048a1596d
353 1795180157.713058 warmup 201 0.04641890525817871 84530fe1b329459ba8c05fbecc725d96 c23fb1ac-67fd-48f5-924d-6fad526fc69c
354 1795180157.960186 warmup 201 0.0545649528503418 b6e31f7e82314adebd2c96abb12976f6 00b781e8-1a6e-4d1a-ae34-75d2a1990179
355 1795180158.2074811 warmup 201 0.024766921997070312 2985a09493b1491c97cae490772bfc8f d02ce7b2-e0a1-439c-8a02-aa57ad954b0e
356 1795180158.4595351 warmup 201 0.1045236587524414 921a1935688f49e5ab1828cd40a3c375 fd12108f-603a-44d5-8d93-d29675978d04
357 1795180158.987233 warmup 201 0.26446008682250977 93913d95e58246299d872da8dedd7920 5aee7bc6-a089-44af-9753-a647806fa999
358 1795180158.708724 warmup 201 0.6227638721466064 7b8b3b12ecde4fbcbeb074dff21b663b cdb7a2a2-2392-4bb6-8feb-18f6c37c22cf
359 1795180159.221353 warmup 201 0.20409274101257324 2a411aca603948868e5b73cda6fe91e5 83299336-71c3-4f85-afd5-9a7d632903be
360 1795180159.463039 warmup 201 0.09139418601989746 26b02c0f71f24e5c8ddc96b9d1ae25a5 c32941b6-a35e-4780-91ed-17cec0715c14
361 1795180159.706288 warmup 201 0.028244972229003906 619307cbe4bf417da6c149d3f3cb5b8c a5fa4e6c-2878-4e01-82b2-678857cfdfa8
362 1795180159.956794 warmup 201 0.012042045593261719 bb655577487a4fc8ace4f902c8e8c199 5ede45e8-2c45-4a1b-a9bd-6f04fdb58c97
363 1795180160.213904 warmup 201 0.05432319641113281 8f7670e1d09241e8937e71bcd4b108d3 f9e8358d-0216-4351-a4cf-8a1b57e1b515
364 1795180160.463031 warmup 201 0.07005786895751953 81fe6f93c621412393a7521026fb86dc de99a316-38a4-4017-af7e-e10e6a4116b0
365 1795180160.705324 warmup 201 0.05199694633483887 7e9577a3bf80456a900e1745534ab5e4 22e6878b-31a2-4364-a94a-54f9daa82365
366 1795180160.956168 warmup 201 0.054311275482177734 88a73eeb3a41494688161931b3f2d0f2 dce46916-4572-4df2-9bdf-70a7b940cd2a
367 1795180161.206967 warmup 201 0.08410191535949707 acf2f4457b724ab5b3da4ab0f57a4c3d 09ef6df9-78f2-4502-80ca-672166481b98
368 1795180161.457957 warmup 201 0.03756880760192871 eeb99aed3aa2474682bb942249b73f96 de89ee75-a4a0-4f50-b7a8-e305cf43126d
369 1795180161.707008 warmup 201 0.023017168045043945 a9263d7d93d04c818df6021d471c3fff 3ec8e5f8-2f11-4cf7-a965-337078d571db
370 1795180161.954315 warmup 201 0.03162097930908203 24367be67ee24f9dabb51672dbc58e04 25172e43-db3c-49a3-8294-187ebc18c469
371 1795180162.207732 warmup 201 0.02358698844909668 de045035ddbc4759a8ff1e8c03e62657 819465ee-7403-43e3-ab30-10db4551195a
372 1795180162.4623191 warmup 201 0.013826847076416016 bed4c0d33e864f52a18dae225d33ed50 b13abc97-d995-4b5c-9749-4e85892946c0
373 1795180162.7049441 warmup 201 0.011426925659179688 37bcbf41ff8a48939d46728863884410 90f819ad-45b1-4e79-b583-45c26033f52e
374 1795180162.95575 warmup 201 0.013515949249267578 f7c1e2922ab84f149cc615144d947801 e9f054a5-969b-4714-b3e6-948247656538
375 1795180163.2059221 warmup 201 0.014194965362548828 7e0523de687c4c358f88f591eb45631d d708f6f8-6747-4a1c-82a8-7056048525a4
376 1795180163.457537 warmup 201 0.012165069580078125 248b9bec4fe34a5d991fd6246acf0c22 13bff701-eab6-4c76-aebb-1b4fe929d1b2
377 1795180163.704688 warmup 201 0.010187864303588867 73f64cbad982461cb68e042fb8b2ef6e 5247ee5c-efad-4a2e-87cf-e955a9c39ce1
378 1795180163.955895 warmup 201 0.01818704605102539 c6c1a0040a2f40d1b729f6e0895f15e8 e5a1c9b2-68fd-4c1f-aeae-4450649c1556
379 1795180164.20519 warmup 201 0.01907491683959961 00e6a3e5c771484dbb237cae0fbfc09d bc2c0346-9643-4d27-a6f7-689b1bed3e85
380 1795180164.4559622 warmup 201 0.02111077308654785 4665df73d5364e5daa3a441143b7c127 a8594acb-4d13-4686-b53b-0346cb860fd5
381 1795180164.713967 warmup 201 0.014435768127441406 e1c3c6397f7f428b85cbcacdb1098b80 b07823c9-ee89-4dbc-ade8-321d3974dc56
382 1795180164.96303 warmup 201 0.021161794662475586 26d86cd296754a089f91b90439e83f60 8ac96c25-a31d-408e-a3fa-b2e6f372a454
383 1795180165.207382 warmup 201 0.025116920471191406 09994b987d8e47898ddad4fb10a09a38 e0d41a3f-d634-42c5-9496-bbecbbe7d733
384 1795180165.463873 warmup 201 0.015794992446899414 df1d6019189446f09582a5bc32757299 7f49017e-5ac2-4521-9772-74d2e55ff1d9
385 1795180165.7129421 warmup 201 0.012481927871704102 cfabbf97b3c14b02bfba1a74acaf4310 add4a145-d07a-49d7-8cc1-5f9774f6d5bf
386 1795180165.961084 warmup 201 0.019973039627075195 5b48ac86de57401eb7f3991cc1904749 7a498621-f882-474a-89a8-486990c9ed2e
387 1795180166.2129421 warmup 201 0.010935068130493164 803f2dd4c81146fd9a5d460b58a5256a f20ae48e-27d6-476e-8207-e99a9e68019a
388 1795180166.454031 warmup 201 0.030212879180908203 b40cfbdfebc044e3b29b3621d95f536a 3df1f455-41e5-4aa7-bfef-8754f1eda936
389 1795180166.712905 warmup 201 0.02558112144470215 00b1bf8353934579a8a95b922f1fdabe ed4d2d9f-b932-40a3-85e5-a9dc6a613e9e
390 1795180166.9541728 warmup 201 0.014276266098022461 9425018ce8324791b8d6eff29d7ee14b 9cfd6c36-ba55-43a4-9d51-13273a3a80df
391 1795180167.213747 warmup 201 0.08199000358581543 b4c47ffe312b4aac87827583c0843fde a414a75c-0936-42af-8715-7013f1e94527
392 1795180167.4605298 warmup 201 0.015784263610839844 84d6ce144a2c40cb95f5ac7690fac5d3 1b856fcc-ff52-4147-90e1-cd94791f7941
393 1795180167.7135592 warmup 201 0.026343822479248047 b7939325d6c34ecea0d35c5d744d3bad 6b28b39b-0f68-4b90-a51f-1c152c58a6c6
394 1795180167.962919 warmup 201 0.015112161636352539 cd5cb0616a7743448c76ce7c9d9b314c 72dfdf21-f5e6-4b74-a1a0-e977ed622969
395 1795180168.2039711 warmup 201 0.011447668075561523 1166a0cbf6474c999930bda4cddd6116 02ab9f26-5ba3-4201-9c58-15aa457967f7
396 1795180168.458719 warmup 201 0.013019084930419922 cbac190c2cc744c1bef9610fec68ddbe c762f7bd-4ea6-4f4a-a59b-fcfa20f2d405
397 1795180168.712939 warmup 201 0.03552412986755371 f4f37c03503344db9ab375351a65cd3e 255cef6f-545f-4ae6-ad6e-99ab8807e772
398 1795180168.957678 warmup 201 0.043527841567993164 b17d2bb8e0714a2d901e1f18637a2814 083f8307-1763-4a1d-b309-0d8d702e5060
399 1795180169.212994 warmup 201 0.0400240421295166 69ac03cd26df493790cfe98a041d5ef6 d0cbec89-cc67-4617-a0d0-26dddb89eae5
400 1795180169.462979 warmup 201 0.01599907875061035 2790773654d84f8d9887e150c9a7d7b3 e67158c3-4809-41f3-b959-9f8c8b06ead7
401 1795180169.712948 warmup 201 0.0437169075012207 4c0ffa1b3d414b4cb5c22e9ce7383a56 58d0c598-8d58-404c-9cea-d3a109fa3886
402 1795180169.9544451 warmup 201 0.20522069931030273 a5a0cfd85436458187897f6cfb92e828 4113d976-439f-4909-9baa-f550c1991184
403 1795180170.2053828 warmup 201 0.044216156005859375 f94451b87409492ba02e70fa8f3877c9 9c45f131-93ff-4678-9ac9-1857ea5116e8
404 1795180170.456413 warmup 201 0.028231143951416016 40ee2a373b2943918e5d19b44780f7c2 b8d0315e-40ff-4b20-a7c4-10eb8a89abd8
405 1795180170.705407 warmup 201 0.14519500732421875 3fe37ad54f6449628ea8ae3b55fabda5 8c124926-7f46-491b-9bf6-9155a58e33c1
406 1795180170.957385 warmup 201 0.1566169261932373 bd234bbdd7a84eef91106d8bb06e4d82 43c16843-8556-4fcf-a412-7c9c85de1829
407 1795180171.209719 warmup 201 0.02808213233947754 1bc7c13575584cc3a996d8e7d4e7130c f59677d7-0cd4-45d7-8bba-4843b1a3b675
408 1795180171.708301 warmup 201 0.03726792335510254 eb4cff5c04cd476db79a908172e69ea4 26149134-587c-4036-8f48-f7e18ef4fe50
409 1795180171.464971 warmup 201 0.3508949279785156 a9a6b04df882480c9bacfdc2a07216b8 5f0fd18b-a430-4f50-9076-e8f8ab1f047a
410 1795180172.206183 warmup 201 0.16024208068847656 7855481b6d0541a087d7f15273a8b09a 247ddeec-8837-4c1d-a60a-2546978fd953
411 1795180171.957116 warmup 201 0.6614320278167725 0963b0b6357f44278d2c3cdfb17e4250 693d8bf9-8e2e-4efc-93ee-be9cbac0b8b4
412 1795180172.4628391 warmup 201 0.2640187740325928 18253581a2bd495aa28106a53393968f c41b1cfa-02d9-43b5-b055-f014f7f57802
413 1795180172.704912 warmup 201 0.03334403038024902 e3a735c71221445fa876a3e44c9cfa47 2ccac5ea-d3ab-4bbb-b063-3b7c36c3ea4e
414 1795180172.9577332 warmup 201 0.06759285926818848 af4bb1940dbe42538d232d83ca9812cc cf812735-2489-4e04-b672-b6143c2c78d0
415 1795180173.206496 warmup 201 0.024734973907470703 70f1582dba2f421ea1980dd833dd5a9d 22898e79-875a-4898-a300-845aa4a6d199
416 1795180173.455349 warmup 201 0.03874802589416504 e858bf8007064349bca1e03aab5eb26f 27cc175e-827a-4188-a728-b78cbc3967c9
417 1795180173.7049768 warmup 201 0.0689091682434082 a066bdd03a4043f3ad62d43d306acdaa c07f5205-97d4-4f59-a9aa-8d05bfc1a52c
418 1795180173.9543931 warmup 201 0.018208980560302734 2e50d32ac368413cb7c4f82c97b55bf1 be9067a9-5d8d-4efb-bb8b-8077e902078e
419 1795180174.2129998 warmup 201 0.012910127639770508 2298a50853b442cf908afff6cd410dda 51aa7305-711e-4137-90b3-34a233cd9587
420 1795180174.46289 warmup 201 0.011858940124511719 fa1e496a8fa54b3bb1168cce8e1a4f23 9f1088f8-a141-4091-a877-fc8d93834d91
421 1795180174.708926 warmup 201 0.013499021530151367 99f56b141a6d499aab6492480f6fa0df d5705025-1ab9-4976-8af8-bec4a632f7f6
422 1795180174.961039 warmup 201 0.05283308029174805 c446fb3191384b2fb20905a33ca1259d 7b9d27c2-7bc5-4768-bbb2-7b433f668401
423 1795180175.2130811 warmup 201 0.046158790588378906 6cac815449fc43ba9ee764b2d3ea89ea b44e4411-77d1-4acb-a6f2-f94fb6b639a1
424 1795180175.459479 warmup 201 0.08919286727905273 00672bb9be9641269cac74d9054f8f7c 27f04e7e-a0fd-4f67-b93b-32560341c85b
425 1795180175.713186 warmup 201 0.07668614387512207 909316d4377046908d3d9b80c5f63734 6c14d788-4b33-4886-8fcc-b87c34433962
426 1795180175.95504 warmup 201 0.02902507781982422 e11701b75c89461f9cf9eb4c49119993 34580b41-aa71-4bf8-935e-4b661f85326e
427 1795180176.2096431 warmup 201 0.023485898971557617 94dca22c54ce4adc9773397279aeeb94 4778e745-e0dd-4ba9-9739-23f228358723
428 1795180176.454023 warmup 201 0.013267993927001953 0c731c2d3c904251a1aaddc02936ac72 b31feb95-00d0-4633-bef1-d3a80d59e132
429 1795180176.712258 warmup 201 0.00846099853515625 5ef7d7287a94477fa7506a8974c8e073 9e76b067-3773-4042-b008-3bcc8ae37289
430 1795180176.9543521 warmup 201 0.008093833923339844 fc63470f48ff40cca32ea6bbd1c87df6 0ef443fe-2d88-4098-828d-5dcd08f0cde5
431 1795180177.204947 warmup 201 0.010506868362426758 f302b68d4cb641eba2ade0caa0d5e7a5 d11c40b9-87d3-4c6c-942d-f0cfb14c72a8
432 1795180177.462801 warmup 201 0.014312028884887695 d62ebff655b64a349542bae704bc43c3 6a603f9c-232b-4fb7-86d2-4d762a4e6e89
433 1795180177.70669 warmup 201 0.011358976364135742 b76a61a69f024f979a61e9ac4206ba23 d252996f-f60b-436b-88e2-5f3f5a19774f
434 1795180177.958333 warmup 201 0.010645866394042969 f81e9d7fe46e4d3f9d2c629a64cf0532 70b74b69-55d0-4b4e-8a63-09f3ca44988b
435 1795180178.21154 warmup 201 0.011842012405395508 d927e2354c124bb1b133252d7ca78ec0 59f1a68a-dc24-4ec1-a940-18e8c75c5544
436 1795180178.456432 warmup 201 0.016075849533081055 fab54f7549c84c32a9333b523bbaf313 4c301fd0-c4ef-41b8-b803-2d0f3d4a47fa
437 1795180178.705439 warmup 201 0.016112804412841797 6795365d631349b8a2b6b0f020fe56d8 f60cab03-ca55-4f74-a133-c5c8ed18c7e3
438 1795180178.957304 warmup 201 0.012420892715454102 76ce4d9cb2d347e7a54f347e1555059e 525440d2-b4bf-4250-9b7b-31e89fb57a27
439 1795180179.21025 warmup 201 0.008681058883666992 d6b683ecea3741df86168a9ca995ea9c cd03fb05-c3b4-4b57-8151-c5e749cfe2c9
440 1795180179.461452 warmup 201 0.015002965927124023 cba9638961a040e3bcb4370420e5a858 f7facafa-db9b-4922-9dc9-e65500737f75
441 1795180179.712847 warmup 201 0.015548944473266602 bf553ad417b040419ae155120de940bc 2aea59d2-0e2f-4929-9651-43073423520d
442 1795180179.960278 warmup 201 0.01565384864807129 ef20979209bb4e448255f580b1a07b61 c20f1a67-e981-4473-b1e0-422fd88447e7
443 1795180180.213103 warmup 201 0.015707015991210938 a32354c47bd845f5bfeb45c98f03c848 9334f89b-289f-4d50-9b87-0afd45cd05aa
444 1795180180.459705 warmup 201 0.012166976928710938 f0fcbc4e2d4b41f098e3429bb23ca236 f856a16e-3469-494a-afd7-943339c4956c
445 1795180180.713276 warmup 201 0.019527196884155273 93e6ed25d7654a8a8aa5a6191c03ae82 a5c347ed-5b9d-4df2-836a-29ddb2d05126
446 1795180180.95435 warmup 201 0.016804933547973633 edae63a4317c42b4a2afde5303f1871a efbff025-41ee-4761-bf4d-3dbe23d4007f
447 1795180181.203885 warmup 201 0.015861034393310547 4531a8464acb49d789e97b7d2cd44f74 79d25593-3116-4f7e-b555-01dacf572512
448 1795180181.4541068 warmup 201 0.010253190994262695 b35a9cb2393442ec88077c5277b69223 cb921d6a-632f-4966-9259-46b8ff7a936b
449 1795180181.712745 warmup 201 0.04211616516113281 b55996e8ad384e92bd22553058887531 0befa854-1659-4b69-82ee-2d9c4452c56e
450 1795180181.962806 warmup 201 0.014417886734008789 eb54c368418c4ea2a81cc55c18f33674 d1339fe5-4fcf-4c51-aa31-ca0d96a21846
451 1795180182.204462 warmup 201 0.012550830841064453 27545433b2c64c5585b1f857b04c41af 2cae5a6a-1c7a-4f5a-92e8-e31936a8deaa
452 1795180182.462759 warmup 201 0.013756036758422852 ba01f205cd36452f91acdcc144693ba3 3ca8359a-17f4-4a02-809a-a0b3258c53bf
453 1795180182.7060359 warmup 201 0.016193151473999023 c9446e7e7bbd4435ac4dd80d99da6237 18f5e9a2-e710-4942-80d9-55604faeef13
454 1795180182.956501 warmup 201 0.022050142288208008 5c19e1f85a624e03a79ed1bb9d338b27 0aeb8bc4-2242-449a-a655-4cbd016f1ee1
455 1795180183.2053308 warmup 201 0.014279365539550781 b35b8690a19f44459906848fc8ea13fb 53bb3ff2-cd49-467c-b323-73ae30e5dafb
456 1795180183.462892 warmup 201 0.05702996253967285 929910e78a024ae092c58d12a1852b24 14309d64-5f24-4db6-b641-3c5e031e1204
457 1795180183.706051 warmup 201 0.03141498565673828 a4e93ec239574d57b20f10bd5e7560ad b6771803-b769-4d57-8b38-1ed6fc6eab85
458 1795180183.960104 warmup 201 0.19160008430480957 646d23e10c5b44f494eb4538753c608b 1bf69a09-4231-4a27-9bfc-6f728053852f
459 1795180184.203552 warmup 201 0.04222893714904785 577e8b16c9cb49b4828a8dd043e749ec 5e8e8000-b2db-4d94-b1c8-0a6636d678f7
460 1795180184.454276 warmup 201 0.014716863632202148 27fd975e87d24480ad32d1ff15d95cae 24f15864-5d6d-4d0c-aed0-8afba06f22b3
461 1795180184.707995 warmup 201 0.03830385208129883 8c2abf5443824c13b1d16a6b7ab8fefd 2cf6a6cc-2a28-4a5b-8c77-7228fde62e7c
462 1795180184.963094 warmup 201 0.01816391944885254 4db1a0d088e549f6ad9b3fe4e3bfca7b 8d0d23ad-6b5b-49f8-a967-0f81fbf04fa4
463 1795180185.213154 warmup 201 0.011205911636352539 277ddd9fe9bd4c798830e9f0b690aa58 940bacab-a245-4c62-85dd-cf7dc89773bf
464 1795180185.458055 warmup 201 0.01757192611694336 89a9630e6b3f4accaa02a5e4fcbfa9b2 0e2796e4-eea9-441e-ba7a-adc3318c4bcd
465 1795180185.717603 warmup 201 0.04112505912780762 0c81fd8100a644d886ab797b31454edf 2a6d1be5-813e-4431-8acb-de28bd6cc485
466 1795180185.9632158 warmup 201 0.014150142669677734 f726becda74545468eccdc64fe022e3c 3310626b-3ff1-49f9-85a3-d46b310862ec
467 1795180186.2084239 warmup 201 0.013811111450195312 ea4985ca5f3340f1878f7790e1d7dee3 bb9f5be6-eee7-4fae-9d7d-708400a5f332
468 1795180186.454124 warmup 201 0.027569055557250977 02b44be192d646bd94c431a7e0cb6ce3 d57d83f0-2dae-4fc2-a1ef-988fe1f370f4
469 1795180186.713099 warmup 201 0.011459112167358398 0dd312b7244a4d5887379d5c0a002e66 6a1c1ae8-f2ee-4851-9b63-6180c5b7b7ec
470 1795180186.956599 warmup 201 0.019530057907104492 92341c3304f3480ca74f390353e35541 54f84857-de49-4ba2-a228-0737e5c4519a
471 1795180187.208894 warmup 201 0.12175798416137695 624595d09b0b4341a6ee3562a5acedc3 c9526170-ffc5-4155-be96-f4afaf01ab28
472 1795180187.4532921 warmup 201 0.030305862426757812 76f2be0fd53c43929d5dbd13c8ea9f4f a14968b7-2b52-4296-8a57-dc2da3cde3e9
473 1795180187.7090461 warmup 201 0.021664857864379883 b8aa6f23adba449cb738ddb921c834bf cc87f9ee-79d8-4bee-8afa-6d9bdc0e50a0
474 1795180187.956699 warmup 201 0.011160135269165039 fb8bc2ed80e248a180abdaac7d680b97 c67baf4c-da95-4b1c-8fd9-54759dd87f7e
475 1795180188.203029 warmup 201 0.014909982681274414 6b91e855e5004849aaa0d0b611cf98f7 44162c07-2165-4353-acea-cbdb32f1f389
476 1795180188.4538748 warmup 201 0.04066634178161621 94c834f81c084b91938fab2d21851366 fb7693b5-7fbc-4675-92cd-3952ec2ecb97
477 1795180188.704454 warmup 201 0.018157005310058594 d759a840803945fe9a93443f2fdd0b2e 4444f578-97a4-4aa9-82c9-a1bd51b35dd7
478 1795180188.963008 warmup 201 0.013981103897094727 f57660a0e9604d7a825a6f5e3743bd94 8577c34d-68b4-4ffb-93ac-4b7bd4c83f21
479 1795180189.203331 warmup 201 0.023202896118164062 244114a321374316adc578862f9200c5 1323a0aa-0302-4608-87b8-55451b318c97
480 1795180189.460938 warmup 201 0.015442848205566406 693f36b531444b85a5ada1bae9069a5d 65fc0dab-bb6b-40e6-9d93-218728ef2118
481 1795180189.7105951 warmup 201 0.010252714157104492 775511c278684cffb821bfdbdbb40ff1 082dea81-4649-43da-af39-8148f5e3e17f
482 1795180189.962133 warmup 201 0.027779102325439453 704f62a681bb4d1d902eeecc72dc8c49 020729eb-6c61-4562-ab6e-4261cbb48cb7
483 1795180190.208668 warmup 201 0.08618903160095215 d58246f4898b413cb3a57efecf51996e 182f947a-cffb-472b-88e4-ae63b9055269
484 1795180190.4556952 warmup 201 0.012355804443359375 85c0ca16930f4f209da3a945ce5c4c0d 48f24757-6028-41af-b1d7-8a4f16f0c467
485 1795180190.712575 warmup 201 0.021033048629760742 8b5ce8b40b5c4735adb4be19e104a00e 75758c4f-8b76-49a1-bf16-396c6d6f074f
486 1795180190.9550889 warmup 201 0.01253509521484375 20981200b4c649a2b3eb03ea7d2953ff 259fa757-9ecc-4169-a160-c9c5b765a199
487 1795180191.212717 warmup 201 0.009023904800415039 18e3ddcd29164515b49ae4df4546ece9 9cd61279-0edd-4c2b-87ad-7da34a443709
488 1795180191.462502 warmup 201 0.01763176918029785 e28e7f1f48144770af9dcffb74421210 272f31c8-dd14-448c-a2b0-0232ead05ddb
489 1795180191.711782 warmup 201 0.015043973922729492 a887526046214c9296d8d7abe36313c9 004511dd-d814-4b80-9979-1e68dfc12f8f
490 1795180191.963177 warmup 201 0.020080089569091797 6e0ffd38e2224d7fa79adfe0bc392818 900a372f-3281-4637-a8d7-8273d340d0f6
491 1795180192.21304 warmup 201 0.03302407264709473 d3b15ced1f164a9eb8b0af6e883a5d0c 5ef64385-53b9-4f95-ae89-866084089856
492 1795180192.4593432 warmup 201 0.026756763458251953 288161a679294cf991c8cf1f700bbee8 5631d6b2-9b07-4d05-be87-8dc2b82dd2a5
493 1795180192.706445 warmup 201 0.02288222312927246 6ff73f860a2d47ada41da4ef9b9d8c1a 11733eb0-f5ef-4e7b-8667-092abf95794a
494 1795180192.962646 warmup 201 0.021658897399902344 d12c7188544d47e6ac8491fab039b363 ac8ad5fd-c7c8-4373-b22b-9a8630e607da
495 1795180193.2061381 warmup 201 0.019336938858032227 075af4e320e344da911d37c4c5a34a91 d54c1b25-df52-4413-a8bf-c00e8fd67c7b
496 1795180193.458136 warmup 201 0.017496109008789062 8c165225b2d64d238b3ad2bd4a696c25 e7fb5585-af0d-41f4-a0b9-a14066005edb
497 1795180193.705204 warmup 201 0.022547006607055664 9599bf2b238a4c78b7a8fda52bc67ff5 e7bb66f1-582a-45c1-b29d-aa4dfcf01619
498 1795180193.9539452 warmup 201 0.26253581047058105 cf4db02bb778433f9ee50fa7fa47cfd5 cc18753c-758e-4be2-b511-9ba6da8872d0
499 1795180194.2069068 warmup 201 0.022910356521606445 cae9dd0a6bf442a4b8e67bad0875c97b 958e8fd7-07f0-4812-8871-ef48157cf903
500 1795180194.460042 warmup 201 0.25853991508483887 885dd8b642104c1fa04f84688d816fc9 f9b1e565-f8e6-4abc-9e7a-65a1c9e4a40c
501 1795180194.7131689 warmup 201 0.07060122489929199 1c56908d205741b2964f718ddb5f3def 69258830-947c-4245-a833-78176422fe70
502 1795180194.952873 warmup 201 0.2982909679412842 8ac2ceb1f42040f8bc6e894fe13abbdc 3dad5269-9754-454e-b9cd-264f86774850
503 1795180195.4555662 warmup 201 0.05453991889953613 3592f8a05103400fa96fcd57e0f5f7c1 c078f8f6-f584-43fd-85df-acabc6b876a9
504 1795180195.207063 warmup 201 0.950869083404541 80b7746064cb43a2adc4f588a84601e5 f0b4607e-5350-45ee-b07f-2be09e95fd99
505 1795180195.70959 warmup 201 0.4569091796875 7b2995e04de8452db02d42aed9eaca94 4cda4fbd-ab10-48bd-88ca-3c48af14315a
506 1795180195.956768 warmup 201 0.22559404373168945 780d3f933edf4e08be22421f5cbb4162 39d0a6bf-c992-4cd2-b4e5-f7796a042a6b
507 1795180196.209187 warmup 201 0.05438494682312012 f5ea813035874d0fb0b08e826ead91e0 fea354b4-7a1c-496c-a804-5e572c0767f3
508 1795180196.454772 warmup 201 0.04239296913146973 bd79b9a297a34dc49a4d6a3b1cba46c2 6a63f12d-a5a1-47d2-bf37-813c326442e3
509 1795180196.706481 warmup 201 0.02643108367919922 615b66824e97490bba5193a10df8fb5e ccc105e9-ebbd-4f96-ad46-85d2ae34e259
510 1795180196.953847 warmup 201 0.024856090545654297 1ddc5ff0885040069f2ebfcbe03bb60c 5d94b5f8-2496-4f7f-8fe2-23c4975a3874
511 1795180197.21262 warmup 201 0.015910863876342773 609bb8bdd431457186d75aac72c48f86 9b98b6b1-31ce-4825-b220-c92fc034d556
512 1795180197.455989 warmup 201 0.03686928749084473 f91f41fc62b54c05b486321f29266270 0c44435a-4ede-4ee1-b83b-4f119384dcaa
513 1795180197.703308 warmup 201 0.02806401252746582 42ecb8cd11ef4e60b95a28d839f19903 8eaf59f5-d4b1-4079-9162-87292d73f31e
514 1795180197.962825 warmup 201 0.012208938598632812 2411d19e7ea942ebbe8d97ceec71bc0b 80a79fa4-e59c-47c1-9048-5cba540a330b
515 1795180198.2126641 warmup 201 0.013143777847290039 faac45e8c35b4e22907ec7a29991e111 f9de74b6-00a9-4736-b224-e50f0e07fca8
516 1795180198.452794 warmup 201 0.010195016860961914 216c854ded9b40ad84c29e4dca3fb40e 8b8b2144-eeb7-4bfb-bcf6-1c3fbd1d3500
517 1795180198.710228 warmup 201 0.01584315299987793 bb930213cbeb48f4be1c535416c3ebef 63588c1d-2d95-4935-9daf-4b6591f718ac
518 1795180198.955477 warmup 201 0.022932052612304688 98735988ba83477c8dc8255e39c6e9be 4f26cb6e-f5e0-43f5-9ff3-00410d4fcb0d
519 1795180199.20573 warmup 201 0.027022123336791992 84770b0f61c3443b9c5b95d8dca08be8 44c5c309-221e-4296-a57b-5a2624514949
520 1795180199.455035 warmup 201 0.0162200927734375 a13929fa98d745b6badf08b6afed734f 616a1684-6a86-455f-9c35-98d63f88a237
521 1795180199.7043462 warmup 201 0.021170854568481445 1a22338792844a6995025059d1573d8b 3fd685cb-bb6c-4b53-a2f2-e67d66e52ecf
522 1795180199.9531882 warmup 201 0.013972997665405273 1803c2fcc5644cd5862fa1c5ba572c5f ada74ea3-81f8-40b1-bb49-62b850483cb6
523 1795180200.212534 warmup 201 0.016909122467041016 f0eb8bba585640e59b164fa58243df6a 95fd836a-e610-40f7-bded-88cf8bbc4d9e
524 1795180200.4543731 warmup 201 0.011936664581298828 cb40183ca2e84009bb204da20c980af0 172837b0-53d7-4056-a9a0-6034f129a3fa
525 1795180200.7096949 warmup 201 0.0234220027923584 399c3e459be045dfb2bae117f1ce2dbd 93cd6edc-c047-4411-befb-6e07e178e52f
526 1795180200.954174 warmup 201 0.02118086814880371 c9fa117ad07a462d84c7b43f8922716a f5fd669c-648f-494b-b836-02a6e00fc0dd
527 1795180201.203704 warmup 201 0.02590489387512207 b23bb2e856984c60b6c634c3f5a1bf8e 231408c4-a640-46ec-b694-ba7b7b26808d
528 1795180201.4537852 warmup 201 0.012441873550415039 293dfa83cfdd475ea7466ed8741d856c eec778a0-e94c-4fc8-960f-98d89e79160a
529 1795180201.7075672 warmup 201 0.009174585342407227 46c8ada6c78149efa0a7ca8e4a5116c2 cdfd8d19-fb43-49dd-8a9f-2ebfec681b55
530 1795180201.953985 warmup 201 0.014340877532958984 fab2d1b470a04bd59bbd3ab89449a39c 5338f15e-4c79-428d-aed7-abb03a0d2e38
531 1795180202.212538 warmup 201 0.024313926696777344 53fac7e2df3e480db30b3d7249691ad2 04d13d7d-12fe-44ec-9185-84397ec920e3
532 1795180202.454725 warmup 201 0.011285781860351562 77d6d1070dac42308a797d848af7d655 7f7e9623-4f82-4732-913b-f5ef7419906c
533 1795180202.704545 warmup 201 0.01698780059814453 d1eca4e404474f548e4c9433975ff7e1 37f34032-000f-497f-9a2d-af4edea502b9
534 1795180202.955153 warmup 201 0.012134075164794922 b3ba6e15f8464b80a44e6b201195a5ce 98fa1948-dcff-4dca-86e4-028ee0edba3d
535 1795180203.208073 warmup 201 0.04192614555358887 a3100e87452b4ea6aeb47bf91637d1a7 d334cda7-3c90-4805-97e8-64b8e2f70a1b
536 1795180203.4534369 warmup 201 0.019451141357421875 42366783be314ea1a699ee7b04915272 882e0ea6-0a04-47d8-84c9-535ff6ba8cb7
537 1795180203.705002 warmup 201 0.019451141357421875 cbe35d7bf653422fbed291fdac71b9d8 fbad4794-3478-4d93-b051-1a7e7aa973ae
538 1795180203.9624739 warmup 201 0.020875215530395508 ae17f67d661349cc8def40a8ef9c50ef 78d6450e-c4ae-42b7-b8d0-e84b45063417
539 1795180204.206752 warmup 201 0.013983011245727539 4cc9c8169d364dce9be1b8f554f1b904 a22865ce-7d25-4d11-80c7-eaab1be90dad
540 1795180204.462346 warmup 201 0.01632404327392578 e7fa08a3cfce42e590897ab29d406ed9 f0a8ceb3-5b07-4e6d-af9e-b268b1ae4f6b
541 1795180204.702373 warmup 201 0.027428865432739258 7fe94400e2124e1f994f4ceee5a86b61 fe1fd31b-25e5-4426-abe7-47d20a216f68
542 1795180204.9624069 warmup 201 0.022263050079345703 1e442c69b70d4c91a2367e58d7c85152 410322ca-61b6-46ad-a236-3403bfd0304f
543 1795180205.2122922 warmup 201 0.00986170768737793 e38450ee6a9843288edccd96a9073daa d2102740-3b08-47cb-bf40-a3c5a083d68d
544 1795180205.4595919 warmup 201 0.012141227722167969 3549a7e31a724d64a9838531e9858237 94382e93-ef4b-4521-b721-fd16cd9a0aa5
545 1795180205.7061708 warmup 201 0.06794524192810059 c7972779afde4c389d99a40a0a1ade6e 7da588d3-2ad5-4b84-a5e2-c5bb86179706
546 1795180205.960969 warmup 201 0.017190217971801758 421aad87ea14472d9fbda6c3dd9ba7d7 4e603bc5-dcb7-4a97-971c-d11de4f95acb
547 1795180206.2061582 warmup 201 0.018009662628173828 d1edf937f3d34f888a97a825cd4349ce 004ce182-0fac-4730-a3f9-70822af82d14
548 1795180206.453884 warmup 201 0.010658979415893555 d760aca1b09a4234aff7b454c14917a3 18dabcdf-5de9-4714-82a2-15c3173e9eec
549 1795180206.707329 warmup 201 0.009528160095214844 33ce61e42af349f9bc78467bd915b819 5ef9abc7-bf6e-4bea-a622-f4a7527aaedf
550 1795180206.95679 warmup 201 0.01139521598815918 739212b4e75f419fab41b1eeec99ac92 16cdc8c3-d46c-47ea-9b71-6f75b08d15ab
551 1795180207.206121 warmup 201 0.0551609992980957 7ffc640f460d4663a2828f833d49a850 a15bc994-c880-44a4-b584-4af86433b876
552 1795180207.453679 warmup 201 0.0324397087097168 a7f33fedba77468e891995294eacbf6c ff7691bd-5d00-42f0-a281-a1268e2795b6
553 1795180207.712791 warmup 201 0.020277976989746094 122ada18bd024ac39b6d09e8a124ba41 351ac8aa-6f70-4695-a389-e20c5c9d6767
+645
View File
@@ -0,0 +1,645 @@
{
"date": "2026-11-20T13:07:47.535044+00:00",
"source_commit": "752a5e6c0d33c2856f7ac8e66074e4adfb9a699d",
"scope": "full30m",
"profile": {
"messages_per_second": 4,
"max_inflight": 16,
"upload_every_seconds": 5,
"image_pixels": [
320,
200
]
},
"complete": false,
"samples": [
{
"at": 1795180074.9828138,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 0.0,
"worker_busy": 0.0,
"worker_attempts": 3.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.023879714074439243,
"slo_good": 9.0,
"slo_bad": 1.0,
"collector_received_spans": 1694.0,
"collector_sent_spans": 1653.0,
"collector_received_logs": 42.0,
"collector_sent_logs": 36.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 606171136.0
},
"alerts": [
{
"name": "ReadySlowBurn",
"state": "pending",
"active_at": "2026-11-20T13:07:09.758725496Z"
}
]
},
{
"at": 1795180083.841643,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 1.736874,
"worker_busy": 0.0,
"worker_attempts": 5.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.16976126432030372,
"slo_good": 10.0,
"slo_bad": 1.0,
"collector_received_spans": 2093.0,
"collector_sent_spans": 2056.0,
"collector_received_logs": 93.0,
"collector_sent_logs": 88.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 550273024.0
},
"alerts": [
{
"name": "ReadySlowBurn",
"state": "pending",
"active_at": "2026-11-20T13:07:09.758725496Z"
}
]
},
{
"at": 1795180093.261457,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 3.1227,
"worker_busy": 0.0,
"worker_attempts": 6.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.15639485593204194,
"slo_good": 11.0,
"slo_bad": 1.0,
"collector_received_spans": 2492.0,
"collector_sent_spans": 2455.0,
"collector_received_logs": 143.0,
"collector_sent_logs": 138.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 550490112.0
},
"alerts": [
{
"name": "ReadySlowBurn",
"state": "pending",
"active_at": "2026-11-20T13:07:09.758725496Z"
}
]
},
{
"at": 1795180102.256551,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 1.905773,
"worker_busy": 0.0,
"worker_attempts": 8.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.11976781488414731,
"slo_good": 13.0,
"slo_bad": 1.0,
"collector_received_spans": 2698.0,
"collector_sent_spans": 2659.0,
"collector_received_logs": 168.0,
"collector_sent_logs": 163.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 550600704.0
},
"alerts": [
{
"name": "ReadySlowBurn",
"state": "firing",
"active_at": "2026-11-20T13:07:09.758725496Z"
}
]
},
{
"at": 1795180111.882427,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 2.421956,
"worker_busy": 0.0,
"worker_attempts": 10.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.10091672235757894,
"slo_good": 15.0,
"slo_bad": 1.0,
"collector_received_spans": 3098.0,
"collector_sent_spans": 3053.0,
"collector_received_logs": 218.0,
"collector_sent_logs": 212.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 539901952.0
},
"alerts": [
{
"name": "ReadySlowBurn",
"state": "firing",
"active_at": "2026-11-20T13:07:09.758725496Z"
}
]
},
{
"at": 1795180121.2686348,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 2.896534,
"worker_busy": 0.0,
"worker_attempts": 12.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.11872534894646448,
"slo_good": 17.0,
"slo_bad": 1.0,
"collector_received_spans": 3492.0,
"collector_sent_spans": 3455.0,
"collector_received_logs": 267.0,
"collector_sent_logs": 262.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 540340224.0
},
"alerts": [
{
"name": "ReadySlowBurn",
"state": "firing",
"active_at": "2026-11-20T13:07:09.758725496Z"
}
]
},
{
"at": 1795180131.386204,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 0.0,
"worker_busy": 0.0,
"worker_attempts": 14.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.10737380018854843,
"slo_good": 20.0,
"slo_bad": 1.0,
"collector_received_spans": 3883.0,
"collector_sent_spans": 3848.0,
"collector_received_logs": 317.0,
"collector_sent_logs": 313.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 540286976.0
},
"alerts": [
{
"name": "ReadySlowBurn",
"state": "firing",
"active_at": "2026-11-20T13:07:09.758725496Z"
}
]
},
{
"at": 1795180140.1158159,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 2.376214,
"worker_busy": 0.0,
"worker_attempts": 16.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.14772727272727285,
"slo_good": 21.0,
"slo_bad": 1.0,
"collector_received_spans": 4279.0,
"collector_sent_spans": 4244.0,
"collector_received_logs": 368.0,
"collector_sent_logs": 363.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 540254208.0
},
"alerts": [
{
"name": "ReadySlowBurn",
"state": "firing",
"active_at": "2026-11-20T13:07:09.758725496Z"
}
]
},
{
"at": 1795180148.943098,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 2.696411,
"worker_busy": 0.0,
"worker_attempts": 18.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.13749999999999962,
"slo_good": 23.0,
"slo_bad": 1.0,
"collector_received_spans": 4680.0,
"collector_sent_spans": 4644.0,
"collector_received_logs": 418.0,
"collector_sent_logs": 413.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 532488192.0
},
"alerts": [
{
"name": "ReadySlowBurn",
"state": "firing",
"active_at": "2026-11-20T13:07:09.758725496Z"
}
]
},
{
"at": 1795180161.9845,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 1.028664,
"worker_busy": 1.0,
"worker_attempts": 19.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.1250000000000001,
"slo_good": 25.0,
"slo_bad": 1.0,
"collector_received_spans": 5027.0,
"collector_sent_spans": 5025.0,
"collector_received_logs": 461.0,
"collector_sent_logs": 461.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 532381696.0
},
"alerts": []
},
{
"at": 1795180173.501747,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 2.296035,
"worker_busy": 1.0,
"worker_attempts": 21.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.2624999999999987,
"slo_good": 27.0,
"slo_bad": 1.0,
"collector_received_spans": 5455.0,
"collector_sent_spans": 5444.0,
"collector_received_logs": 518.0,
"collector_sent_logs": 516.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 533110784.0
},
"alerts": []
},
{
"at": 1795180182.0484211,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 1.632452,
"worker_busy": 1.0,
"worker_attempts": 23.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.30208333333333304,
"slo_good": 29.0,
"slo_bad": 1.0,
"collector_received_spans": 5838.0,
"collector_sent_spans": 5834.0,
"collector_received_logs": 566.0,
"collector_sent_logs": 564.0,
"collector_failed_spans": 0.0,
"collector_queue": 1.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 533004288.0
},
"alerts": []
},
{
"at": 1795180191.050949,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 2.37788,
"worker_busy": 0.0,
"worker_attempts": 26.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.26499999999999924,
"slo_good": 31.0,
"slo_bad": 1.0,
"collector_received_spans": 6244.0,
"collector_sent_spans": 6198.0,
"collector_received_logs": 618.0,
"collector_sent_logs": 612.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 533053440.0
},
"alerts": []
},
{
"at": 1795180202.356919,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 0.84311,
"worker_busy": 0.0,
"worker_attempts": 28.0,
"pool_used": 1.0,
"pool_waiting": 0.0,
"http_p95": 0.2662500000000005,
"slo_good": 33.0,
"slo_bad": 1.0,
"collector_received_spans": 6632.0,
"collector_sent_spans": 6593.0,
"collector_received_logs": 667.0,
"collector_sent_logs": 662.0,
"collector_failed_spans": 0.0,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 0.0,
"app_memory": 532992000.0
},
"alerts": []
}
],
"phases": [
{
"name": "warmup",
"start": 1795180069.957184,
"planned_seconds": 300
}
],
"elapsed_seconds": 140.295,
"requests_csv": "slo-research-first-requests.csv",
"request_statuses": {
"201": 552
},
"uploads": [
[
1795180069.957095,
"warmup",
202,
"a4530828-8dfc-4109-b628-c02a3798ccf1",
"6b01f3f7a2c341fa8671f4d7a5d53fac"
],
[
1795180074.9614038,
"warmup",
202,
"c7630b1f-ae32-4307-85ce-1ad2b9d2ce82",
"57b638b29f49450e8d1638504c6586c5"
],
[
1795180079.971358,
"warmup",
202,
"2fe8450f-20bb-43db-bbea-e440571f46b4",
"863d1ebc8de645fca91cca1b39d7dd5d"
],
[
1795180084.971487,
"warmup",
202,
"3b99ce14-4b93-4446-a8cd-d19cf1032345",
"30150967491048afbccffa77cbe1438d"
],
[
1795180089.980097,
"warmup",
202,
"b91d2c9f-66fb-47bb-b4da-3a65110d24ea",
"a7740e5a4e634f0e8eb508adfe0ad3fb"
],
[
1795180094.99019,
"warmup",
202,
"03975eb4-f8cc-453c-a291-381d1213063d",
"7e86598bfe5942f5910a9076c4451f13"
],
[
1795180099.991049,
"warmup",
202,
"67468f17-bac2-4546-8190-f5c21a12d099",
"83aa89ae7b084fa7b44e2cee426f2b55"
],
[
1795180104.9930298,
"warmup",
202,
"12874504-b542-4906-9cd2-bb52d9128539",
"736eec0cee79414fb1d20a5ae6622f7f"
],
[
1795180109.997144,
"warmup",
202,
"6f17982e-907c-442f-972d-4156261e0603",
"a94be328ecf34e7da17789c6a7fa0902"
],
[
1795180115.0068,
"warmup",
202,
"4ddac804-25bb-4d31-b491-c34a66f834ef",
"da4430b7065842f2a089cea6db3d8d22"
],
[
1795180120.0094888,
"warmup",
202,
"39b39e5c-0750-442c-8d87-14dfdbedeea4",
"e54443c722124a3284515f71b24cb006"
],
[
1795180125.0124538,
"warmup",
202,
"ae29301a-cd3c-49a9-88c7-f96d42eff4d3",
"d64a7beb706844038d898951bb059c2f"
],
[
1795180130.01914,
"warmup",
202,
"a3d56ad4-c89d-4c09-880d-effd96fd2fc7",
"982067e9b4bf4dfd9c72ac87c8054086"
],
[
1795180135.029181,
"warmup",
202,
"ebe4358b-e000-4341-bcda-5ccb5f4016b9",
"3625b14d985242efbdd86fde7db2d2cc"
],
[
1795180140.030897,
"warmup",
202,
"39507a43-8917-4c79-b3cf-31931e82813f",
"e93edc815c2540a280185aa9a5e57ec0"
],
[
1795180145.038631,
"warmup",
202,
"78c052c3-1321-429b-b69e-994fbbc04738",
"d9f9d2bdf0064f97bed896362b878e50"
],
[
1795180150.043772,
"warmup",
202,
"df3768d5-8727-4083-8e0b-280c278b9a5e",
"cbe3f59ea86e4917bcdd5dffae7fa391"
],
[
1795180155.116708,
"warmup",
202,
"38e19499-2f53-429f-9f39-696f8ddf9d78",
"d5b874084df34ceda4a536b8704e3577"
],
[
1795180160.1169658,
"warmup",
202,
"fd1ab7f5-fefe-463b-97f7-2b909cd314d2",
"0a6d63e813c3456c9e26f14863b13067"
],
[
1795180165.117381,
"warmup",
202,
"aef650a9-1573-4d0d-9a9c-8562e9e3b544",
"5749949cae4f47e59d8f5b4800937e05"
],
[
1795180170.123688,
"warmup",
202,
"e52a9186-83c3-40bd-9f0c-05dc56a653d8",
"1922cb0439244afe9050bca802cae6e1"
],
[
1795180175.127093,
"warmup",
202,
"ca907578-1425-4536-91cf-8bf7a39645ba",
"16041423c09e4c6cab733a7c939543ce"
],
[
1795180180.128174,
"warmup",
202,
"ee979ea1-bf9e-4ea0-9e99-9af941030d60",
"501f6d91de9e4c6bab4dc0c9b574745e"
],
[
1795180185.137634,
"warmup",
202,
"bd9a05f7-e2a5-4113-9d2c-805ee37be00a",
"b8fb5b6793ac457ba6ef67830fb3a3eb"
],
[
1795180190.142824,
"warmup",
202,
"a7d987c0-f74c-467c-8e77-129e1741d733",
"47ce6dee6f8e45a9b6107b59170dfa0e"
],
[
1795180195.147142,
"warmup",
202,
"a8b31d86-325a-4398-a14e-d406b6644355",
"16760f23381d4a788f486ff1600e47e3"
],
[
1795180200.147872,
"warmup",
202,
"7eea348e-417f-4cf3-8097-73211b00d1ef",
"af7fe6002e514612a625962d360970d9"
],
[
1795180205.1478279,
"warmup",
202,
"7de6cd57-c67f-4076-9972-7f3a539a5f79",
"46ecb113b6314b928232eb5bfece1cad"
]
]
}
@@ -0,0 +1,10 @@
Traceback (most recent call last):
File "<frozen runpy>", line 203, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "<repository>/scripts/research_slo.py", line 631, in <module>
main()
~~~~^^
File "<repository>/scripts/research_slo.py", line 479, in main
max(s["signals"]["collector_memory"] for s in points)
< report["container_limits"]["otel"]["mem_limit"]
TypeError: '<' not supported between instances of 'float' and 'str'
@@ -0,0 +1,223 @@
timestamp,phase,status,seconds,trace_id,message_id
1795180306.423019,warmup,201,0.055117130279541016,76b17926effb4139b48ade994fd1b7de,45edeac4-7cee-4570-b117-9b26c569b543
1795180306.674056,warmup,201,0.11263298988342285,c6b3b19c02944693b96797163527546f,f9e4dcbc-dae1-4e5c-b8a2-74edf26846cb
1795180306.925218,warmup,201,0.024322986602783203,3eb10d8793dd4669ac70599f8be9eb90,0a3a27a5-1178-4faf-a664-4702712e783e
1795180307.181909,warmup,201,0.06856274604797363,3a0245c1ee554cbeb5ad64de9ba6e75c,102398b3-cfcb-4686-b2d8-10994305cc1d
1795180307.4298592,warmup,201,0.050598859786987305,f2bd58199b5245a685b5fda701ecff3d,48d12936-0916-4eaa-aaf9-05b2c364c87f
1795180307.6729841,warmup,201,0.016305923461914062,ff9fb6fe48aa4a87bcdc079e014d31b6,ef71bafd-8db5-4b6c-9c67-d2f630135491
1795180307.9231732,warmup,201,0.0525507926940918,14318f14c2c8420ab8d4d3ae3e0bcf40,ace23a9f-f216-4282-b3ae-2a7985d8f37d
1795180308.180491,warmup,201,0.034638166427612305,10944fea4ef94741ac5dde89779d2783,9f34c3fe-40fb-4d81-8140-5722ad929244
1795180308.422879,warmup,201,0.0330810546875,584c358735fc4175b803475f58b3670e,83dae3dc-900a-48e8-8bdd-2dca4a879315
1795180308.672776,warmup,201,0.02603316307067871,92fa42f3977b423ea6405fe6c84108c5,c58e9f22-94dc-4b9b-aa6f-e58fa4bd4ce5
1795180308.927267,warmup,201,0.021506071090698242,16b96bd7db7648ad81c67d33af6da58e,5b0e326e-ad2b-4a9a-ba40-89e16ac238cf
1795180309.178731,warmup,201,0.04657888412475586,b894046a5e24431db9777227f00e5e5e,d76724d8-0b6f-4014-9331-5104e76286ce
1795180309.426372,warmup,201,0.01767587661743164,9c914e9234234f6b9ec5aebd0700b08d,e19cfe8c-439f-4094-b088-78a5fa598a49
1795180309.6781561,warmup,201,0.034906864166259766,aff7ab332dd44350a77bbe170ea2254c,bd24569d-e245-47a4-8713-61982c26b8e6
1795180309.923274,warmup,201,0.06194877624511719,bafc8d9e97124b0788c1d0ebd258e03b,bcec2bbb-2014-4d77-a740-35c279e875ab
1795180310.1744561,warmup,201,0.27138471603393555,792f1a0bbbcb40439b4908a661f2453b,cdd7e2ce-bae9-4430-9f77-21dba2d63759
1795180310.4293928,warmup,201,0.03658699989318848,62b9f45e808e4001b8d94f32a951d09a,d4a2ef1b-2bb1-4d75-9c31-6b029bdb1cfb
1795180310.682456,warmup,201,0.027057886123657227,89af845a6be644b6a3052232976be957,4c2264e0-0b23-4884-bd2d-72aa066894c9
1795180310.931253,warmup,201,0.017241239547729492,e5c8386c382c4b3e9ead235dc2581b04,7f90ab2b-c35b-4ce1-a813-7320024cd8d1
1795180311.177042,warmup,201,0.0361180305480957,ecd9c03f4e884f4b82acbea5cb6612d2,faa61c28-259f-40e4-aec8-ecb33dfbdac3
1795180311.426838,warmup,201,0.0326380729675293,f686d484d73448a3b95729eac184bf66,7c150bc6-edbd-4e39-befa-b60f3d905972
1795180311.6764312,warmup,201,0.15946173667907715,b9dde373785c4e048354ed3ce67d3300,db20ab6a-f67d-495c-9fc8-d8df2b123fe0
1795180311.929388,warmup,201,0.044622182846069336,ba4dfdcff45b4c53a7a07904fa61f94a,f6be3558-4eee-459f-9014-3838e8393b1c
1795180312.1749911,warmup,201,0.07517695426940918,957ff476cdbd41b290da286dfcc60b16,fe032f0d-e2c9-4550-afdc-a1d099385366
1795180312.4307652,warmup,201,0.020746946334838867,dbfdb96f8ac44c58b1160e0fa42a6984,c4538789-36b3-43bc-9d95-b56d017ea9c5
1795180312.67829,warmup,201,0.06154799461364746,a905d5b3feaa437e91ad1299ae1ed7c0,1d9a4f48-5605-4d6d-8021-9cc92ab58f45
1795180312.928323,warmup,201,0.011504888534545898,972030cfa8694a50a7f8abbafb7a8f87,dadee304-20e6-42df-9108-46b53634cb8e
1795180313.181609,warmup,201,0.014528036117553711,5a58df298b354208bac9300c340649d6,d9e7e3ce-2ea0-4948-804d-c28f16f047b7
1795180313.428823,warmup,201,0.021541833877563477,7e4a6d7bb2b3441bbaa56c17f376bd1c,942cdccf-9cbb-473b-aca0-f6018b42aad8
1795180313.682583,warmup,201,0.011409759521484375,2ab48b9886874f7ab4b224e5e0fb47df,0ab859a3-61dd-42d6-b32d-f2dd92129bc1
1795180313.926015,warmup,201,0.09591507911682129,25a7bbaad50445f99471f406b5afb9a4,538117fb-17dc-4dbd-8708-82a431a28102
1795180314.174363,warmup,201,0.022596120834350586,a0366368908c441e9a0d41543390ef78,37e7f867-90be-4c6d-97b4-250ce02b1328
1795180314.4236832,warmup,201,0.01119089126586914,1c295becc4334f6dbfd8448b1c7106a4,948c8e31-2d92-4a2e-94b0-67a47144368e
1795180314.680267,warmup,201,0.01999187469482422,e512543e7f7d44ce960de76da5b17cc3,0614e562-4376-468e-b84e-76e904a4218c
1795180314.929228,warmup,201,0.019069910049438477,b7a3683322474a8a8f066d0664b9d431,175c9e9d-eff8-4e12-8ec2-27c3450f0f0f
1795180315.180566,warmup,201,0.015458106994628906,cf5a580d5c08459996d3d8231b807ad4,127952d0-8a07-48a8-b807-26dc230ae1a9
1795180315.425687,warmup,201,0.018053770065307617,22942e117b2e4d958b71083c877ef01b,aae0dbb8-a3a9-457a-a0d0-68bae6960d7f
1795180315.675526,warmup,201,0.026155948638916016,721fe3c04e7b480394d2d6ddaa8e1a6a,53dd64c2-b25f-4ae9-a170-a840e3ce21bc
1795180315.924555,warmup,201,0.011884927749633789,7f06f138058e4a1484b29ade9a22eba5,27a4defe-4e15-4ecd-bc31-ed2c75d18d83
1795180316.1811168,warmup,201,0.032523155212402344,0e0aae0cd32a42d9b962da5117e36666,b0d5c7c9-60e5-4d86-b4f2-703fa9bfdcc6
1795180316.4279091,warmup,201,0.018095970153808594,a6f9a0c0acd244b69d902cb79725d373,573c9fbf-70c9-4fa1-a992-727904d1e568
1795180316.673577,warmup,201,0.009534835815429688,94cefc2182544e3981144d87d4d4fcee,47ffbbd9-a1ad-40c6-8466-884c598607ca
1795180316.926636,warmup,201,0.014694929122924805,5bbef0350fb64b6bac71b79770ea13f0,52d9bb5b-d73d-4952-8c59-446b262912e1
1795180317.182466,warmup,201,0.015308141708374023,74e404a445284782b86afc86fe4644b5,25f85804-c3f2-4585-a2ea-cdee0c7fc7d2
1795180317.428221,warmup,201,0.010089874267578125,3495bd89b7ad4f349515b64c09374f81,6b7d028b-30e7-49db-ae12-4abca73e816a
1795180317.678735,warmup,201,0.014368057250976562,192e67e8fd5244589801a03e0771ec4a,13aade2d-bba3-44ed-83f2-179be7eb03b0
1795180317.9327989,warmup,201,0.022715330123901367,b0f30a0f95c84b15823b2bac102c8d0f,df64a7a8-fa8c-4743-9c35-a2711710139d
1795180318.1822689,warmup,201,0.011677265167236328,d7ea7d3721e645278cff25c11e34d4de,ebc15bda-434d-479e-8951-0e4bb696082a
1795180318.4249558,warmup,201,0.02953815460205078,7989769fa68546c89900ef7023cdfe52,a66b2086-50c8-4c87-a3b8-63f4d8aeca8d
1795180318.678638,warmup,201,0.013857126235961914,d8cd43459576487dad26c79c97c2292a,05a4b084-9a87-47e4-873d-349fdd3a4643
1795180318.9239738,warmup,201,0.014153242111206055,00d13065595845af8b4b55052390c819,cfb927dc-00a9-45dd-9734-aaa4f1f0cb22
1795180319.1802251,warmup,201,0.0406038761138916,e10654f795ae4419a4b812aadb96970e,f3accec8-f106-413c-ac14-a63a943aa726
1795180319.429311,warmup,201,0.01441502571105957,19b74567fbe747fdb24d7d06efa6d1eb,2ceb1a29-52d1-42a1-9565-78acfd5e9fc8
1795180319.6822212,warmup,201,0.008408784866333008,93cb6031e83747bc920fc2fb9610661f,5b60fc86-3a0a-4df9-96a9-b2631210d432
1795180319.923309,warmup,201,0.03721809387207031,7ef202b3302c499ca0910961fd7a3433,aed217bd-aaff-44d5-b5bc-b704d0e37234
1795180320.1749089,warmup,201,0.060726165771484375,38326ddf0825495a8902d615734748bc,444bac02-07bf-4e3f-8e88-f7abc4b3d347
1795180320.4323468,warmup,201,0.027462244033813477,a334076530fd46cb9c146528fc43c6a8,64d62ba8-4512-41ef-bcda-2e5e33e996ec
1795180320.6773481,warmup,201,0.03374075889587402,0b2a7694ca2b45ba912289c2deedce69,0f8b0cce-bf64-48ff-a038-7a2d3a32c280
1795180320.924204,warmup,201,0.013178825378417969,3454ac5fb3304fe7b53bc8445b0d05d1,4bb1b2fb-e11e-4a28-9330-040f6cb8d25c
1795180321.174137,warmup,201,0.010026931762695312,dbf03db4b1d8440587d5a1f60d5ea14f,09e09bf5-02a6-451f-82a8-769e6aa0facd
1795180321.428226,warmup,201,0.011801958084106445,ca76f22c791748d48455ca5f7381661d,0a7de369-03a2-44eb-8c15-b1c53e13fbc8
1795180321.679889,warmup,201,0.010977029800415039,dd6f8475d4ed4376868c4405935692fd,35fee93a-8bd8-4824-9822-6ca26266e5b0
1795180321.9291892,warmup,201,0.011880874633789062,6096d515e8f1422f81419c5fb1a124ef,bfb8f12f-9be1-4089-8b93-d2eef7625675
1795180322.180209,warmup,201,0.0462651252746582,9b12c16b30024558b2f47cecfddf1f82,d7ac3615-aafa-4ee6-86a3-7cd9497fb093
1795180322.422112,warmup,201,0.05217790603637695,2c18bdf8d4ca47ec97d0279c968e2ea6,c308ad03-e992-4ef0-8ca0-8faf408f1650
1795180322.68032,warmup,201,0.01891303062438965,a9088dcd6c5b4fb2993d9b8131868f04,8fab0bdc-1125-4e0f-9645-e040744f5730
1795180322.924831,warmup,201,0.011192083358764648,363cada73f8e40b3863498d521f84103,61491b7d-7e26-4ad3-af54-759104015bfc
1795180323.182313,warmup,201,0.06261301040649414,b868e760d7434f16b95e07fc3c036e03,494387fb-8db0-494c-b720-a63d0d52eeb5
1795180323.432374,warmup,201,0.01579904556274414,e8e2992ee027446a99d04e7ed54cc5fb,1d69df6e-a939-44bd-9cc4-0e1e87ccaab4
1795180323.682173,warmup,201,0.02521991729736328,11093f1c7d764d229815e1b41b456396,da181d3e-a345-4a02-96c7-463d144f0a96
1795180323.925667,warmup,201,0.016142845153808594,6d3e0801fec94b5da2834f7a736bf56f,16f9b4e6-f3fc-4c0d-8432-2e81ec0bcc13
1795180324.174363,warmup,201,0.0512690544128418,1d99895f9e434de3843e0248703eb1e4,89d42939-6925-4cd8-88ff-288e1dc85428
1795180324.4281669,warmup,201,0.021039247512817383,c8b92c7d5d56452faa59c4d2ba09197a,57b8846b-51f2-4d28-9a88-14eb08a73817
1795180324.674187,warmup,201,0.021563053131103516,78665560fdef454788a688ff19ba0ee4,e3945170-8574-4883-a3d6-73ed17abdc56
1795180324.926713,warmup,201,0.09740400314331055,eefc3558c2a54ec19a5bdcbffd5872ff,4b3ff7a6-26bc-4a4f-b6f5-5b20e27e8ba9
1795180325.179863,warmup,201,0.0670158863067627,10d8be6f1b094b4a860825c5cdcb66bf,1abc8596-c125-448f-b6bd-f56e5db13cb8
1795180325.430054,warmup,201,0.07341194152832031,baf1cf9b30174a0eb160a4d564f15813,cf8c046c-7e9e-4448-90dd-dd294ebad9ac
1795180325.6817129,warmup,201,0.009120941162109375,6a862f33186c4cebbb25ff98178f2bfa,95527733-2c5a-498d-9f49-f8d301d4d1c3
1795180325.926674,warmup,201,0.012025117874145508,cd82070cc11f4ef29b911375577865c3,486bf416-ff9b-4597-bbbd-dc75a7c30490
1795180326.182048,warmup,201,0.013367891311645508,5eebee0de5624d7985d7565367d7c6b4,5945a92a-1095-49a5-9d13-29097b3f622c
1795180326.428158,warmup,201,0.013689041137695312,f9d615eb908743ecb21146371ff6f4c2,d52b0441-fbeb-43e8-a95f-08c455160db5
1795180326.672092,warmup,201,0.010310173034667969,1b7f0b149b4c4b1dbd9904657ebf0ae4,af5aa204-1c19-48de-bb9a-c78fb8290f9e
1795180326.9302368,warmup,201,0.015584230422973633,c3b77b73036944bcb20956e0ba2f6a90,ebc74447-e1ea-4c86-8e0d-697d5195b2db
1795180327.173146,warmup,201,0.01581406593322754,90ba92757b584b6fbe0358a4df665ab0,6c928cc6-7a0f-4b4a-901e-edd2b8a5c83f
1795180327.427222,warmup,201,0.012076139450073242,d1daf4f998454f4ea6d0297bda8f6e04,607a7eed-38ac-4965-9478-6895a83e8fdf
1795180327.672278,warmup,201,0.010951995849609375,9ec736838ed949fb83a63200454fea36,0d74bd92-7493-43c9-a354-0c65d3b97e97
1795180327.9319308,warmup,201,0.011237144470214844,fb5eaf9967284cb397cbceae4661dddd,9e8c6eb6-ce68-43c6-9e92-8de1057ce3b1
1795180328.175093,warmup,201,0.01266622543334961,35270abcaff847faa9429abd5c4de7e6,aae33024-4f87-4108-8cd6-63ff8766e9a7
1795180328.423997,warmup,201,0.13264989852905273,b9780a4af98a41baaccbb8106999db5d,80ae627d-9a2d-4e82-b729-4f707b1f21c7
1795180328.6788118,warmup,201,0.045153141021728516,d5ed1030e5bf401caa1f2a5adc0873cb,4dfd91fd-2d14-4ad9-9114-de44974aac8b
1795180328.922585,warmup,201,0.014145135879516602,9a3d7c283964490089d1c6d261185589,62e1ba19-45d2-41a8-b841-a043110f7704
1795180329.178997,warmup,201,0.022284984588623047,8951f43b75ec4005a7608f31c70c26a6,beb8b635-f948-4dbd-8ff5-1265fde89b09
1795180329.424947,warmup,201,0.04422593116760254,9240a763e0e94a62b52ebfc4ad708cc4,0c6428e3-de7b-4c53-b446-a3665b141b7f
1795180329.680489,warmup,201,0.03175187110900879,7f8fc03fa51e4dd2a37339ac606ffecb,fd540e12-8e6f-4374-ab1e-d52b34d78e4e
1795180329.9236531,warmup,201,0.013557672500610352,98c5a9f09f044dcbbff494387bafbd4c,93589d91-8865-4cf8-8bad-58564a889d89
1795180330.1782389,warmup,201,0.02124500274658203,1e695d9d2c2f48b88dfd76f8bf8cf490,5d160787-23f3-4e62-94a3-cdf9b282a871
1795180330.423929,warmup,201,0.03000617027282715,9d4e91db41fc44d2891be75670dfabc9,44d47b59-ba1f-4c2a-9d33-9791ea4db19c
1795180330.6799638,warmup,201,0.21182918548583984,6cbf676363d54de9bbf98522a6457cc8,63f7279b-586e-4fa4-9dcd-702c937f806a
1795180330.9291131,warmup,201,0.020792722702026367,7fd6e1c7fbc644bfab8092c898d748e7,672e9274-7ab2-4667-b35e-af3bd8b2b036
1795180331.179322,warmup,201,0.046911001205444336,021915e1b41b4f34ad85f3d2e6a4da0f,dce01763-b702-4ce9-9ee3-921e4b3c9886
1795180331.4241679,warmup,201,0.05442523956298828,99a3588c6c91443bbf6bbc2cdcac7d2a,29cc6791-e6fe-4f9e-992f-4479215fc266
1795180331.678278,warmup,201,0.017962217330932617,99d25a5eb2064395927eb09e8aa222cf,8a16a0e9-0219-44e8-a920-653874d0b8d9
1795180331.92304,warmup,201,0.018758058547973633,cffa0a7dd9bc4236a6038010bf40cccc,ea7f502a-39f9-418e-9fea-7906409ed598
1795180332.176664,warmup,201,0.012223958969116211,559e362406b84895a79a3cbaf54d49c2,713a7dce-5626-404d-9662-121e875c8528
1795180332.421847,warmup,201,0.015980958938598633,20d2755def3649c98662f8e7142e2681,415f4a48-ebdf-4511-abe1-deabbc7d73a9
1795180332.67451,warmup,201,0.029402971267700195,ae57e3232d8240baa8a8cce8ab650325,54a8e098-a1f5-4b6a-9e43-0b056bb3e2e6
1795180332.930043,warmup,201,0.011388063430786133,cee81cf842a544e2b1700c6d5e5f4cbd,731cf770-d0eb-4aec-8aed-f5949290a301
1795180333.172213,warmup,201,0.01586604118347168,bf8d441585fd49b788c92dbfd279d25c,47cafb93-d453-47e7-85d2-ead4ee14c814
1795180333.423415,warmup,201,0.014913082122802734,aee265c80d76491ca511c2dd67e5bef7,4e051465-0aed-418b-874d-2fa35311e668
1795180333.67177,warmup,201,0.011767864227294922,b5b423c76c1a43fb8883ea4801d396dc,42fdaa69-7e20-43a3-8b59-7a638f9621b6
1795180333.928659,warmup,201,0.013628959655761719,5c0752fad40d4761bfde52ee54822e58,afc00df1-4d20-412c-872c-d0d320cf3d37
1795180334.175783,warmup,201,0.07439613342285156,478b50c50a3f4fe59b7aa4962d110637,ad0585cc-a72f-49dd-8545-bb0935f21245
1795180334.428273,warmup,201,0.013356924057006836,b0ffff00bf0a4f02a7e8de3e83944f0d,1c24e631-7a77-498e-a625-8538d60f62e5
1795180334.672097,warmup,201,0.013913869857788086,be440f8a05d3492a9a4816f7aa2c02a0,04df633a-94bf-4348-b397-ac7b59937040
1795180334.929533,warmup,201,0.6379430294036865,d8e4bcffdbd24e47ad264517c65f0dfa,6963c7af-7de7-4700-89b3-4c4bd02de125
1795180335.4310532,warmup,201,0.21014761924743652,e8a7cc4c3dba45daba6cabedfdbaabf0,87999bae-e07d-42e8-86c0-4a457dfabf0b
1795180335.179592,warmup,201,0.46358299255371094,198efae63dd14b83a1f67c8256b36fb2,2d5926f4-0437-4983-80f7-93f33a3c2dda
1795180335.674002,warmup,201,0.05605912208557129,c30042701c1f4f1eaef7bdc88b3920f7,f8a28688-8d60-4dcb-8d19-754ec9d586d6
1795180335.929294,warmup,201,0.00999593734741211,38366b0cc61c42fda055656fc8b9405c,e8f7ea7c-71f2-4463-bdff-d16a901bb8f5
1795180336.1780539,warmup,201,0.012089014053344727,b1ef2220cce54480a9236bed82c24ff5,2d1c4a5c-7368-42ee-9fd1-f84cf3207222
1795180336.424557,warmup,201,0.01134800910949707,0561752001b1407a9da166b663bee655,763b99c1-ded3-4502-82f3-508af1b4b232
1795180336.681586,warmup,201,0.012669086456298828,b2507ead5d2f41548c5513101f01c681,a70bf5b4-cacd-408c-83b8-f73d8b9c6663
1795180336.92322,warmup,201,0.025698184967041016,42c5d403c13243e48ab3497efd64efe1,d1f46a4a-7792-4438-878e-b172420013ae
1795180337.175337,warmup,201,0.025685787200927734,732a1d48efac4a8182e803fae21b14b0,f8d80429-8488-4674-9a2c-164ff01ec638
1795180337.4312031,warmup,201,0.012318849563598633,551a0437a6684e04858e3e9e18e2e27a,449a474d-4d08-44bb-8773-023f2ffe5ebc
1795180337.676717,warmup,201,0.011137008666992188,58204786011c47978fc1a737c82d2ee0,740dc698-cbfc-4352-a7d4-4eb229b12cd8
1795180337.928061,warmup,201,0.010760784149169922,d5beaf3bbcd441fc93b3614c408f70c7,7477d17c-61fc-42a6-8192-e361298e1050
1795180338.1814332,warmup,201,0.008683919906616211,8da12b3fd0d84ca2b7e9b8b8399e6689,55629cb3-8f72-47ce-8870-02d6f8fb2e11
1795180338.4314299,warmup,201,0.012953042984008789,dc03191b57b740c7ba16770974d28edc,8802a764-3bb3-4474-a456-aa16351ad526
1795180338.674549,warmup,201,0.011281728744506836,501a10a665cd41ff9ca6f1bdf261b436,4af67448-4e3b-4100-a970-6aa2c28a60b4
1795180338.921731,warmup,201,0.035794973373413086,9458f35691df40d6817b1cbc7884da09,a9b1b50c-3ecb-4e40-a233-4a84efd565f4
1795180339.176345,warmup,201,0.01884603500366211,2424406ec5d043259e8ac310ab765f6e,0bdea318-c103-4d5e-b476-274c4feaef9b
1795180339.428579,warmup,201,0.011592864990234375,50aaf29486064b24b9bc50e652f46cb9,edc06291-9fb4-42bb-a501-d6c1bf6f09a0
1795180339.680001,warmup,201,0.009851932525634766,c6dce8e46aac45b8aefdefee3ef007f1,5a335b34-fef6-44bf-bda2-6a1b5482d587
1795180339.931456,warmup,201,0.010718822479248047,8b8b563bf97a4a9e9eb24d1e3f1585f2,ff857b95-1b74-4f86-b405-f7314690b2d8
1795180340.1814861,warmup,201,0.011487007141113281,7187e721be584781a50e5c622995bec9,f0a80079-1e90-4407-a747-a00800379415
1795180340.4302,warmup,201,0.019636869430541992,b91a363608904d27bd4d5f4cd801b2f3,b658efea-ec64-41d6-b268-73810fc09974
1795180340.671793,warmup,201,0.015497922897338867,cb3272c4ea774827b839828948ca594a,be030a11-9ee2-4ee3-9f20-e0e124351756
1795180340.9280891,warmup,201,0.010939836502075195,c3f948a801ce48e1be201fb0a71efbbc,9190bf06-a491-489d-83e3-c12a518ad08c
1795180341.1780741,warmup,201,0.013393878936767578,57c094a056fb4901bcdbd9f98cb84035,64f9cb7e-3bec-42bd-976f-22f463257144
1795180341.423362,warmup,201,0.01970696449279785,f45ebf0ce50c4728a612832e45620dfa,82024df4-278e-4726-84e1-ed4f4c5ab451
1795180341.6748412,warmup,201,0.008790969848632812,ae5a9f1e20fa4882b0190f619832f229,123057d3-1867-4be0-97f9-81638814ae9e
1795180341.92869,warmup,201,0.08092522621154785,cf7231989bff434eb919c723499df91b,f68c3ae0-ca25-4b9d-ad9d-d8ab866a052d
1795180342.175219,warmup,201,0.011922836303710938,095db27bed494b6780624dcd0ab805cb,847c10dc-bcc6-42ae-b2a6-c1cf38aad0ff
1795180342.424076,warmup,201,0.011336803436279297,9db82bcc7a2349e6a028a579b59f3731,2ac0cbf5-c8a5-4dfc-9f00-9e53cfe74ba1
1795180342.671635,warmup,201,0.012029886245727539,4b4531114e8842118cb57552bfe8580b,c48917dd-b068-4a03-b8f8-c8ba300ea444
1795180342.930646,warmup,201,0.03807401657104492,77c386ecd5ad42138dbf01ed9a1a08e3,530fe2ca-019f-4498-b28a-1b46003b42fd
1795180343.173782,warmup,201,0.02783679962158203,9fa8bd612584480b8b8d54f9d97f84c6,71a2ea1a-a408-4b76-813c-6cc112518100
1795180343.431467,warmup,201,0.015299081802368164,ea11b713e9a34a16aae02e94ad918c04,8a705499-b818-4059-bf44-c742cc33385d
1795180343.681278,warmup,201,0.014876127243041992,06e399d5a290466fab9e5d633b60dd0a,4dcb497e-4b72-4523-9898-a2700a14d355
1795180343.923985,warmup,201,0.01096200942993164,cc43e56b4a2b4c19b2b6101a50c2bc94,48490016-195a-4a81-8f04-ba808859df9c
1795180344.171476,warmup,201,0.03735709190368652,e39608915fcb42debbe456471d2da56d,6fbc4730-e650-470b-bfc9-ffc4617929af
1795180344.4280179,warmup,201,0.020684003829956055,cef374c710c2463281482742a9d4f9d5,562998c0-4619-476d-9d87-0205e598889a
1795180344.6780381,warmup,201,0.06757974624633789,3699dadfb0c146679e5b1d18cf0ab47d,0ffc8be8-a8e3-42c4-8315-c078810a2ff7
1795180344.930443,warmup,201,0.01471090316772461,97b7b81f51994c4e9e513b189a6ffd37,a7a27dc8-522e-4567-961c-0db57fdea6db
1795180345.173086,warmup,201,0.015903949737548828,68b24d7b4e6e4d16ae9506091f733fb3,c63ccb84-a500-423f-861a-6de057114f5b
1795180345.42819,warmup,201,0.01735401153564453,31f7ff52721e4e61aa77a46238e37f5d,0a9ec936-7f43-45fb-8b0f-b66d7f4a7476
1795180345.676356,warmup,201,0.02088189125061035,d1f916e4f1934981a8fd1c3884230435,5ebfe76f-62a3-4ed8-ab48-710d781cf530
1795180345.92798,warmup,201,0.012233972549438477,dda6e0e723374b63a94ec455b1842d6e,aa9b085d-60fc-474b-a417-e6da81114221
1795180346.174879,warmup,201,0.020541906356811523,00d03e3aab9e43efb45b1b99a77dd1e5,62f165a2-06ff-4919-862c-463502cd33eb
1795180346.4241002,warmup,201,0.01257777214050293,a9631cd4218448598816db46e05ac210,1a359611-1102-4cd0-a8a8-19f739c2d5f6
1795180346.6805272,warmup,201,0.031142711639404297,d6326c50cf6a4db4a14a959f047aec67,229641cd-c43f-42bb-bceb-a636fcc2e025
1795180346.923172,warmup,201,0.013450145721435547,fc9078ab07b6497da577535c92e91ceb,85107c40-5384-48dc-bf32-18b48892a006
1795180347.1747231,warmup,201,0.05333566665649414,07f0bbeaa6b44c61a7c351107b41b237,4ae310e9-fa62-43c6-b1d5-5330e738b353
1795180347.427953,warmup,201,0.014930009841918945,202c8996cce944b9a22892a908097a34,53140b2b-a88d-4fa6-899a-a17775f5dfaf
1795180347.680614,warmup,201,0.023784160614013672,907d1193824c42e5b30c45ba2634e043,ec60f205-beb2-4c00-9e47-1dd4c7a421fb
1795180347.926591,warmup,201,0.010828018188476562,6adcea95ee4c4ed1a2ece269f55dbedf,156e019b-22eb-4dd0-bd16-10af4e5d41a4
1795180348.173774,warmup,201,0.03443479537963867,f67d7dc601194d20af0034748e2c835d,a2e2b4c1-d602-4451-990c-3480b5861430
1795180348.424813,warmup,201,0.025806903839111328,32f93effa4e745e6ba519405061bf598,1d99fe38-930b-4373-b8b1-c7a50e5883d4
1795180348.675287,warmup,201,0.019309043884277344,ae8cd26f20be4a35b3f63d7b46a8b960,d31bebc4-a124-4ae3-abc2-af41e2d5eae6
1795180348.923858,warmup,201,0.010105133056640625,bc0cc3806d7f4a919e5f47cda40b0cb4,19928ca1-9013-4dc2-b4d6-72ac6a1953e8
1795180349.177574,warmup,201,0.018340110778808594,786e7738d9c748d9809bd84ff34b95c4,4642a90b-25bd-491c-bf74-e6bb9890d5d6
1795180349.429085,warmup,201,0.04597902297973633,b1166b143286416284b919b7ee8beb1d,415730d0-e983-454a-9099-29502d67c8e4
1795180349.678473,warmup,201,0.12787294387817383,73a76ddd1f284f46af12cb353cc71ada,7197c218-1cc3-46a9-b2c0-77200ea16b4b
1795180349.921549,warmup,201,0.009884834289550781,d5c670be06a745ffbb46333a26e750f6,63e5afa5-b537-4bef-b6f9-02467ead6dad
1795180350.180325,warmup,201,0.014764785766601562,163c2435e5b84ac3a6d9925094a8896b,136194ea-2088-486c-87d4-d0b7644f17f5
1795180350.424948,warmup,201,0.015136957168579102,c06a3f595cf54357a11aa8e6e99ca18b,8385080b-1cc6-44c5-a89b-85b8244591de
1795180350.67535,warmup,201,0.01176905632019043,ae155b69f38b40efa06dc335484ce719,5d981da2-e2b0-4f58-945b-4e4f4fecd654
1795180350.928941,warmup,201,0.020803213119506836,27cf380a14bf461eb619d155d979b8f4,8c4a855a-39a7-4ef8-bbbd-90e03768e88c
1795180351.17329,warmup,201,0.018284082412719727,45247b8b0c484db4a857c5575a1da514,60f682a1-f799-4439-8f9a-082cbe158558
1795180351.423593,warmup,201,0.02286505699157715,140ac94fb75846ccbc27e9e4dff67afa,f854fece-effa-4ab5-974b-72f86703bcac
1795180351.678071,warmup,201,0.024228811264038086,e1ab03f2591b4971a7f90801486f2bf2,609aa3e0-7e44-496f-b64c-bbc1f9fa99a9
1795180351.923866,warmup,201,0.01363992691040039,88c05ce549ef41c5a3a14c95c53bd409,365710a3-5f35-4adc-9c25-af3fe51493dd
1795180352.177956,warmup,201,0.03061199188232422,21889320216743caa5c1392d19903af3,43dbfb78-3ebe-407b-b7e1-63e8cb3b9b5a
1795180352.429983,warmup,201,0.012333154678344727,9de8eb1a06574abb826292e19b84f988,92e9a6bd-88fa-4610-b6c0-b5349308b4b0
1795180352.674007,warmup,201,0.01972508430480957,f0ce8192797a4645b76b7e5808f2192f,6b22227e-67c4-4a9a-a0ca-08a8ef20e693
1795180352.930673,warmup,201,0.014127016067504883,d3157bed5a4346baaec4b82e25019ecd,358d7a14-0c3d-45d2-be95-0630c878db02
1795180353.172828,warmup,201,0.016306161880493164,48bb51fe3a604a499d067faf39701e9b,6ab03add-9f65-4731-a343-d154bf2a40e9
1795180353.428106,warmup,201,0.021203994750976562,1583b7683f8141df8bb9883c0a1f189d,d3aa43a2-e707-41c2-8f1a-e3e1e7634623
1795180353.677095,warmup,201,0.015395164489746094,d222f39f6a9a4d2e8f540d9b27aad707,320dfd89-7d56-42b7-b01c-cf04d8b513cd
1795180353.9265192,warmup,201,0.08537173271179199,6a936c0664b74374a9f2187a2ddbbf06,708acef1-ae2b-4229-b4cc-9d19a2a9c57e
1795180354.1799161,warmup,201,0.02597975730895996,c6bd819ca872476e94666e03b7f3992c,b3877af3-258b-4c77-8172-72b0bbee5b27
1795180354.423289,warmup,201,0.010989904403686523,d30989f1247f4de7bfd4b9c32888d35c,09d175b4-c5be-4ded-bdea-47fee06baceb
1795180354.674009,warmup,201,0.07170987129211426,1d4784d0947541f897b9279180f8d08f,2aac78f9-62f4-4f09-8d2d-526b9b21c9b0
1795180354.924747,warmup,201,0.11100912094116211,ae9f5b183ecb4a67a1e29e88ecd685ca,837b2f21-0baa-4fc1-bd69-e53168ec033b
1795180355.171475,warmup,201,0.181365966796875,929df48dbefa466dad31befb9d377330,e47a3bea-d950-47df-ac26-483c12bafe44
1795180355.4301581,warmup,201,0.1972649097442627,275af00105a34802986c50739697c1ac,52788b1c-9221-4af9-8684-5b55a7aaf92a
1795180355.673067,warmup,201,0.08739495277404785,39a13ad72b4448f0a767afb20028d7fc,59aa1c2d-b9f9-4634-af5c-c1977cb58871
1795180355.92431,warmup,201,0.04182100296020508,a5a63d8af2ae4bedb575dc8739f3af2f,f5b3aac5-9941-4e03-a70e-e6e5e522a110
1795180356.179839,warmup,201,0.06274628639221191,46d5eeb65b6d45ef9a81ec1369b7dd39,edbe7d82-fd87-423a-b49e-e395f740eb2c
1795180356.427257,warmup,201,0.014807939529418945,f2d123f7377c4b0cbd1649a3785a2b90,ce02ec3a-c8a8-4123-a1ad-cc29835e8825
1795180356.674825,warmup,201,0.011126041412353516,679d40c9ee6c481c81b28ce99f0e42b6,6c636c07-672e-4a2a-9058-db745c3463f0
1795180356.927894,warmup,201,0.018886804580688477,552981d06df34d508ebd5846bb836a83,ffd07a80-f592-4ddc-a3cc-ff2a9f197fb6
1795180357.1745431,warmup,201,0.009415864944458008,f79131e7537e4f0f8350b6fdcb0b96da,bec88e0d-4f2e-44b4-8b37-91062206723c
1795180357.426614,warmup,201,0.04293084144592285,ae7be590ae294f2dac57e987ede2b4f6,df77674e-3378-4778-97d7-c89c7b2fa738
1795180357.680987,warmup,201,0.12665009498596191,4b4003e700d546d1a66698887fa1380c,8a809c0b-b8cb-4074-ac85-8e7a43dca87e
1795180357.9307442,warmup,201,0.022862672805786133,4fb7f73760a445dbac480366c1f672e5,6e063f18-7f51-4b19-8ed0-a0a7d73f95aa
1795180358.1806982,warmup,201,0.037358760833740234,57b36f91a50a4302b67b697e8f8f08a4,0e1ea419-593b-4ca3-ac11-723e02b24ff3
1795180358.423149,warmup,201,0.029945850372314453,603c36cdc55144618038a00cec0e2aec,e6102a10-97ca-432b-beb3-fa9f02501717
1795180358.680745,warmup,201,0.01798725128173828,87ae437cafe647e29098811ddd6df769,b769fee9-17a2-49af-9d99-ac04d0201df4
1795180358.921466,warmup,201,0.015487909317016602,f8c239845d61458d975549317f54a085,7e568898-e1ff-436c-8c80-673f25a8db5f
1795180359.174216,warmup,201,0.016328811645507812,b052f222a93f4b7fbdef7bc74f89f2d2,bbf7df75-90b7-4ec7-abef-10d8ccce8c6c
1795180359.4299269,warmup,201,0.0165250301361084,25510d762ac3496ba3eeb873580d0df5,9287de57-3881-4e80-93fd-7d2e550cab75
1795180359.674637,warmup,201,0.018573760986328125,a4a36dd2e72d41c5af7dac81276b78f3,474e577f-89a4-485f-ae33-43dc9e0073b2
1795180359.928923,warmup,201,0.036789894104003906,c48bf0ba1acb4dfeb92a33a5f7c6cf31,780b5f8d-4b0f-4ec5-ae82-f21d1727f87a
1795180360.178257,warmup,201,0.2533531188964844,49c5d7b7dfc448e98f41e786cbdd2998,a0474355-8589-42f4-9094-7d8163b02b12
1795180360.421695,warmup,201,0.052854061126708984,ff183e56bb5c4ac18f6d6f3859cc5100,0d099a99-b8b0-467d-91b4-e608e2d8f1d4
1795180360.670847,warmup,201,0.06912517547607422,d40d667c826e49a8840ba9184578117c,bb64c924-9ff3-43d9-b2b2-4441497f8878
1795180360.928828,warmup,201,0.08232808113098145,840f9a5f453249ba8c47882f57488674,8adf8c34-21aa-4000-829f-3775b3b5bcbf
1795180361.1724942,warmup,201,0.17403888702392578,8c867b70da9649899232fe5772fc5df3,69a8b185-8eec-46fc-9c3c-1d857076d989
1795180361.427998,warmup,201,0.021191835403442383,250cf8f6e9d54242abd898c562c6b5c9,67dae0d0-333d-4f9a-b909-014e5396107d
1795180361.678324,warmup,201,0.024866104125976562,4d702fdc4a814531844a755099df5f46,2de43ad9-4cfc-40a3-a168-069a77fbe13f
1 timestamp phase status seconds trace_id message_id
2 1795180306.423019 warmup 201 0.055117130279541016 76b17926effb4139b48ade994fd1b7de 45edeac4-7cee-4570-b117-9b26c569b543
3 1795180306.674056 warmup 201 0.11263298988342285 c6b3b19c02944693b96797163527546f f9e4dcbc-dae1-4e5c-b8a2-74edf26846cb
4 1795180306.925218 warmup 201 0.024322986602783203 3eb10d8793dd4669ac70599f8be9eb90 0a3a27a5-1178-4faf-a664-4702712e783e
5 1795180307.181909 warmup 201 0.06856274604797363 3a0245c1ee554cbeb5ad64de9ba6e75c 102398b3-cfcb-4686-b2d8-10994305cc1d
6 1795180307.4298592 warmup 201 0.050598859786987305 f2bd58199b5245a685b5fda701ecff3d 48d12936-0916-4eaa-aaf9-05b2c364c87f
7 1795180307.6729841 warmup 201 0.016305923461914062 ff9fb6fe48aa4a87bcdc079e014d31b6 ef71bafd-8db5-4b6c-9c67-d2f630135491
8 1795180307.9231732 warmup 201 0.0525507926940918 14318f14c2c8420ab8d4d3ae3e0bcf40 ace23a9f-f216-4282-b3ae-2a7985d8f37d
9 1795180308.180491 warmup 201 0.034638166427612305 10944fea4ef94741ac5dde89779d2783 9f34c3fe-40fb-4d81-8140-5722ad929244
10 1795180308.422879 warmup 201 0.0330810546875 584c358735fc4175b803475f58b3670e 83dae3dc-900a-48e8-8bdd-2dca4a879315
11 1795180308.672776 warmup 201 0.02603316307067871 92fa42f3977b423ea6405fe6c84108c5 c58e9f22-94dc-4b9b-aa6f-e58fa4bd4ce5
12 1795180308.927267 warmup 201 0.021506071090698242 16b96bd7db7648ad81c67d33af6da58e 5b0e326e-ad2b-4a9a-ba40-89e16ac238cf
13 1795180309.178731 warmup 201 0.04657888412475586 b894046a5e24431db9777227f00e5e5e d76724d8-0b6f-4014-9331-5104e76286ce
14 1795180309.426372 warmup 201 0.01767587661743164 9c914e9234234f6b9ec5aebd0700b08d e19cfe8c-439f-4094-b088-78a5fa598a49
15 1795180309.6781561 warmup 201 0.034906864166259766 aff7ab332dd44350a77bbe170ea2254c bd24569d-e245-47a4-8713-61982c26b8e6
16 1795180309.923274 warmup 201 0.06194877624511719 bafc8d9e97124b0788c1d0ebd258e03b bcec2bbb-2014-4d77-a740-35c279e875ab
17 1795180310.1744561 warmup 201 0.27138471603393555 792f1a0bbbcb40439b4908a661f2453b cdd7e2ce-bae9-4430-9f77-21dba2d63759
18 1795180310.4293928 warmup 201 0.03658699989318848 62b9f45e808e4001b8d94f32a951d09a d4a2ef1b-2bb1-4d75-9c31-6b029bdb1cfb
19 1795180310.682456 warmup 201 0.027057886123657227 89af845a6be644b6a3052232976be957 4c2264e0-0b23-4884-bd2d-72aa066894c9
20 1795180310.931253 warmup 201 0.017241239547729492 e5c8386c382c4b3e9ead235dc2581b04 7f90ab2b-c35b-4ce1-a813-7320024cd8d1
21 1795180311.177042 warmup 201 0.0361180305480957 ecd9c03f4e884f4b82acbea5cb6612d2 faa61c28-259f-40e4-aec8-ecb33dfbdac3
22 1795180311.426838 warmup 201 0.0326380729675293 f686d484d73448a3b95729eac184bf66 7c150bc6-edbd-4e39-befa-b60f3d905972
23 1795180311.6764312 warmup 201 0.15946173667907715 b9dde373785c4e048354ed3ce67d3300 db20ab6a-f67d-495c-9fc8-d8df2b123fe0
24 1795180311.929388 warmup 201 0.044622182846069336 ba4dfdcff45b4c53a7a07904fa61f94a f6be3558-4eee-459f-9014-3838e8393b1c
25 1795180312.1749911 warmup 201 0.07517695426940918 957ff476cdbd41b290da286dfcc60b16 fe032f0d-e2c9-4550-afdc-a1d099385366
26 1795180312.4307652 warmup 201 0.020746946334838867 dbfdb96f8ac44c58b1160e0fa42a6984 c4538789-36b3-43bc-9d95-b56d017ea9c5
27 1795180312.67829 warmup 201 0.06154799461364746 a905d5b3feaa437e91ad1299ae1ed7c0 1d9a4f48-5605-4d6d-8021-9cc92ab58f45
28 1795180312.928323 warmup 201 0.011504888534545898 972030cfa8694a50a7f8abbafb7a8f87 dadee304-20e6-42df-9108-46b53634cb8e
29 1795180313.181609 warmup 201 0.014528036117553711 5a58df298b354208bac9300c340649d6 d9e7e3ce-2ea0-4948-804d-c28f16f047b7
30 1795180313.428823 warmup 201 0.021541833877563477 7e4a6d7bb2b3441bbaa56c17f376bd1c 942cdccf-9cbb-473b-aca0-f6018b42aad8
31 1795180313.682583 warmup 201 0.011409759521484375 2ab48b9886874f7ab4b224e5e0fb47df 0ab859a3-61dd-42d6-b32d-f2dd92129bc1
32 1795180313.926015 warmup 201 0.09591507911682129 25a7bbaad50445f99471f406b5afb9a4 538117fb-17dc-4dbd-8708-82a431a28102
33 1795180314.174363 warmup 201 0.022596120834350586 a0366368908c441e9a0d41543390ef78 37e7f867-90be-4c6d-97b4-250ce02b1328
34 1795180314.4236832 warmup 201 0.01119089126586914 1c295becc4334f6dbfd8448b1c7106a4 948c8e31-2d92-4a2e-94b0-67a47144368e
35 1795180314.680267 warmup 201 0.01999187469482422 e512543e7f7d44ce960de76da5b17cc3 0614e562-4376-468e-b84e-76e904a4218c
36 1795180314.929228 warmup 201 0.019069910049438477 b7a3683322474a8a8f066d0664b9d431 175c9e9d-eff8-4e12-8ec2-27c3450f0f0f
37 1795180315.180566 warmup 201 0.015458106994628906 cf5a580d5c08459996d3d8231b807ad4 127952d0-8a07-48a8-b807-26dc230ae1a9
38 1795180315.425687 warmup 201 0.018053770065307617 22942e117b2e4d958b71083c877ef01b aae0dbb8-a3a9-457a-a0d0-68bae6960d7f
39 1795180315.675526 warmup 201 0.026155948638916016 721fe3c04e7b480394d2d6ddaa8e1a6a 53dd64c2-b25f-4ae9-a170-a840e3ce21bc
40 1795180315.924555 warmup 201 0.011884927749633789 7f06f138058e4a1484b29ade9a22eba5 27a4defe-4e15-4ecd-bc31-ed2c75d18d83
41 1795180316.1811168 warmup 201 0.032523155212402344 0e0aae0cd32a42d9b962da5117e36666 b0d5c7c9-60e5-4d86-b4f2-703fa9bfdcc6
42 1795180316.4279091 warmup 201 0.018095970153808594 a6f9a0c0acd244b69d902cb79725d373 573c9fbf-70c9-4fa1-a992-727904d1e568
43 1795180316.673577 warmup 201 0.009534835815429688 94cefc2182544e3981144d87d4d4fcee 47ffbbd9-a1ad-40c6-8466-884c598607ca
44 1795180316.926636 warmup 201 0.014694929122924805 5bbef0350fb64b6bac71b79770ea13f0 52d9bb5b-d73d-4952-8c59-446b262912e1
45 1795180317.182466 warmup 201 0.015308141708374023 74e404a445284782b86afc86fe4644b5 25f85804-c3f2-4585-a2ea-cdee0c7fc7d2
46 1795180317.428221 warmup 201 0.010089874267578125 3495bd89b7ad4f349515b64c09374f81 6b7d028b-30e7-49db-ae12-4abca73e816a
47 1795180317.678735 warmup 201 0.014368057250976562 192e67e8fd5244589801a03e0771ec4a 13aade2d-bba3-44ed-83f2-179be7eb03b0
48 1795180317.9327989 warmup 201 0.022715330123901367 b0f30a0f95c84b15823b2bac102c8d0f df64a7a8-fa8c-4743-9c35-a2711710139d
49 1795180318.1822689 warmup 201 0.011677265167236328 d7ea7d3721e645278cff25c11e34d4de ebc15bda-434d-479e-8951-0e4bb696082a
50 1795180318.4249558 warmup 201 0.02953815460205078 7989769fa68546c89900ef7023cdfe52 a66b2086-50c8-4c87-a3b8-63f4d8aeca8d
51 1795180318.678638 warmup 201 0.013857126235961914 d8cd43459576487dad26c79c97c2292a 05a4b084-9a87-47e4-873d-349fdd3a4643
52 1795180318.9239738 warmup 201 0.014153242111206055 00d13065595845af8b4b55052390c819 cfb927dc-00a9-45dd-9734-aaa4f1f0cb22
53 1795180319.1802251 warmup 201 0.0406038761138916 e10654f795ae4419a4b812aadb96970e f3accec8-f106-413c-ac14-a63a943aa726
54 1795180319.429311 warmup 201 0.01441502571105957 19b74567fbe747fdb24d7d06efa6d1eb 2ceb1a29-52d1-42a1-9565-78acfd5e9fc8
55 1795180319.6822212 warmup 201 0.008408784866333008 93cb6031e83747bc920fc2fb9610661f 5b60fc86-3a0a-4df9-96a9-b2631210d432
56 1795180319.923309 warmup 201 0.03721809387207031 7ef202b3302c499ca0910961fd7a3433 aed217bd-aaff-44d5-b5bc-b704d0e37234
57 1795180320.1749089 warmup 201 0.060726165771484375 38326ddf0825495a8902d615734748bc 444bac02-07bf-4e3f-8e88-f7abc4b3d347
58 1795180320.4323468 warmup 201 0.027462244033813477 a334076530fd46cb9c146528fc43c6a8 64d62ba8-4512-41ef-bcda-2e5e33e996ec
59 1795180320.6773481 warmup 201 0.03374075889587402 0b2a7694ca2b45ba912289c2deedce69 0f8b0cce-bf64-48ff-a038-7a2d3a32c280
60 1795180320.924204 warmup 201 0.013178825378417969 3454ac5fb3304fe7b53bc8445b0d05d1 4bb1b2fb-e11e-4a28-9330-040f6cb8d25c
61 1795180321.174137 warmup 201 0.010026931762695312 dbf03db4b1d8440587d5a1f60d5ea14f 09e09bf5-02a6-451f-82a8-769e6aa0facd
62 1795180321.428226 warmup 201 0.011801958084106445 ca76f22c791748d48455ca5f7381661d 0a7de369-03a2-44eb-8c15-b1c53e13fbc8
63 1795180321.679889 warmup 201 0.010977029800415039 dd6f8475d4ed4376868c4405935692fd 35fee93a-8bd8-4824-9822-6ca26266e5b0
64 1795180321.9291892 warmup 201 0.011880874633789062 6096d515e8f1422f81419c5fb1a124ef bfb8f12f-9be1-4089-8b93-d2eef7625675
65 1795180322.180209 warmup 201 0.0462651252746582 9b12c16b30024558b2f47cecfddf1f82 d7ac3615-aafa-4ee6-86a3-7cd9497fb093
66 1795180322.422112 warmup 201 0.05217790603637695 2c18bdf8d4ca47ec97d0279c968e2ea6 c308ad03-e992-4ef0-8ca0-8faf408f1650
67 1795180322.68032 warmup 201 0.01891303062438965 a9088dcd6c5b4fb2993d9b8131868f04 8fab0bdc-1125-4e0f-9645-e040744f5730
68 1795180322.924831 warmup 201 0.011192083358764648 363cada73f8e40b3863498d521f84103 61491b7d-7e26-4ad3-af54-759104015bfc
69 1795180323.182313 warmup 201 0.06261301040649414 b868e760d7434f16b95e07fc3c036e03 494387fb-8db0-494c-b720-a63d0d52eeb5
70 1795180323.432374 warmup 201 0.01579904556274414 e8e2992ee027446a99d04e7ed54cc5fb 1d69df6e-a939-44bd-9cc4-0e1e87ccaab4
71 1795180323.682173 warmup 201 0.02521991729736328 11093f1c7d764d229815e1b41b456396 da181d3e-a345-4a02-96c7-463d144f0a96
72 1795180323.925667 warmup 201 0.016142845153808594 6d3e0801fec94b5da2834f7a736bf56f 16f9b4e6-f3fc-4c0d-8432-2e81ec0bcc13
73 1795180324.174363 warmup 201 0.0512690544128418 1d99895f9e434de3843e0248703eb1e4 89d42939-6925-4cd8-88ff-288e1dc85428
74 1795180324.4281669 warmup 201 0.021039247512817383 c8b92c7d5d56452faa59c4d2ba09197a 57b8846b-51f2-4d28-9a88-14eb08a73817
75 1795180324.674187 warmup 201 0.021563053131103516 78665560fdef454788a688ff19ba0ee4 e3945170-8574-4883-a3d6-73ed17abdc56
76 1795180324.926713 warmup 201 0.09740400314331055 eefc3558c2a54ec19a5bdcbffd5872ff 4b3ff7a6-26bc-4a4f-b6f5-5b20e27e8ba9
77 1795180325.179863 warmup 201 0.0670158863067627 10d8be6f1b094b4a860825c5cdcb66bf 1abc8596-c125-448f-b6bd-f56e5db13cb8
78 1795180325.430054 warmup 201 0.07341194152832031 baf1cf9b30174a0eb160a4d564f15813 cf8c046c-7e9e-4448-90dd-dd294ebad9ac
79 1795180325.6817129 warmup 201 0.009120941162109375 6a862f33186c4cebbb25ff98178f2bfa 95527733-2c5a-498d-9f49-f8d301d4d1c3
80 1795180325.926674 warmup 201 0.012025117874145508 cd82070cc11f4ef29b911375577865c3 486bf416-ff9b-4597-bbbd-dc75a7c30490
81 1795180326.182048 warmup 201 0.013367891311645508 5eebee0de5624d7985d7565367d7c6b4 5945a92a-1095-49a5-9d13-29097b3f622c
82 1795180326.428158 warmup 201 0.013689041137695312 f9d615eb908743ecb21146371ff6f4c2 d52b0441-fbeb-43e8-a95f-08c455160db5
83 1795180326.672092 warmup 201 0.010310173034667969 1b7f0b149b4c4b1dbd9904657ebf0ae4 af5aa204-1c19-48de-bb9a-c78fb8290f9e
84 1795180326.9302368 warmup 201 0.015584230422973633 c3b77b73036944bcb20956e0ba2f6a90 ebc74447-e1ea-4c86-8e0d-697d5195b2db
85 1795180327.173146 warmup 201 0.01581406593322754 90ba92757b584b6fbe0358a4df665ab0 6c928cc6-7a0f-4b4a-901e-edd2b8a5c83f
86 1795180327.427222 warmup 201 0.012076139450073242 d1daf4f998454f4ea6d0297bda8f6e04 607a7eed-38ac-4965-9478-6895a83e8fdf
87 1795180327.672278 warmup 201 0.010951995849609375 9ec736838ed949fb83a63200454fea36 0d74bd92-7493-43c9-a354-0c65d3b97e97
88 1795180327.9319308 warmup 201 0.011237144470214844 fb5eaf9967284cb397cbceae4661dddd 9e8c6eb6-ce68-43c6-9e92-8de1057ce3b1
89 1795180328.175093 warmup 201 0.01266622543334961 35270abcaff847faa9429abd5c4de7e6 aae33024-4f87-4108-8cd6-63ff8766e9a7
90 1795180328.423997 warmup 201 0.13264989852905273 b9780a4af98a41baaccbb8106999db5d 80ae627d-9a2d-4e82-b729-4f707b1f21c7
91 1795180328.6788118 warmup 201 0.045153141021728516 d5ed1030e5bf401caa1f2a5adc0873cb 4dfd91fd-2d14-4ad9-9114-de44974aac8b
92 1795180328.922585 warmup 201 0.014145135879516602 9a3d7c283964490089d1c6d261185589 62e1ba19-45d2-41a8-b841-a043110f7704
93 1795180329.178997 warmup 201 0.022284984588623047 8951f43b75ec4005a7608f31c70c26a6 beb8b635-f948-4dbd-8ff5-1265fde89b09
94 1795180329.424947 warmup 201 0.04422593116760254 9240a763e0e94a62b52ebfc4ad708cc4 0c6428e3-de7b-4c53-b446-a3665b141b7f
95 1795180329.680489 warmup 201 0.03175187110900879 7f8fc03fa51e4dd2a37339ac606ffecb fd540e12-8e6f-4374-ab1e-d52b34d78e4e
96 1795180329.9236531 warmup 201 0.013557672500610352 98c5a9f09f044dcbbff494387bafbd4c 93589d91-8865-4cf8-8bad-58564a889d89
97 1795180330.1782389 warmup 201 0.02124500274658203 1e695d9d2c2f48b88dfd76f8bf8cf490 5d160787-23f3-4e62-94a3-cdf9b282a871
98 1795180330.423929 warmup 201 0.03000617027282715 9d4e91db41fc44d2891be75670dfabc9 44d47b59-ba1f-4c2a-9d33-9791ea4db19c
99 1795180330.6799638 warmup 201 0.21182918548583984 6cbf676363d54de9bbf98522a6457cc8 63f7279b-586e-4fa4-9dcd-702c937f806a
100 1795180330.9291131 warmup 201 0.020792722702026367 7fd6e1c7fbc644bfab8092c898d748e7 672e9274-7ab2-4667-b35e-af3bd8b2b036
101 1795180331.179322 warmup 201 0.046911001205444336 021915e1b41b4f34ad85f3d2e6a4da0f dce01763-b702-4ce9-9ee3-921e4b3c9886
102 1795180331.4241679 warmup 201 0.05442523956298828 99a3588c6c91443bbf6bbc2cdcac7d2a 29cc6791-e6fe-4f9e-992f-4479215fc266
103 1795180331.678278 warmup 201 0.017962217330932617 99d25a5eb2064395927eb09e8aa222cf 8a16a0e9-0219-44e8-a920-653874d0b8d9
104 1795180331.92304 warmup 201 0.018758058547973633 cffa0a7dd9bc4236a6038010bf40cccc ea7f502a-39f9-418e-9fea-7906409ed598
105 1795180332.176664 warmup 201 0.012223958969116211 559e362406b84895a79a3cbaf54d49c2 713a7dce-5626-404d-9662-121e875c8528
106 1795180332.421847 warmup 201 0.015980958938598633 20d2755def3649c98662f8e7142e2681 415f4a48-ebdf-4511-abe1-deabbc7d73a9
107 1795180332.67451 warmup 201 0.029402971267700195 ae57e3232d8240baa8a8cce8ab650325 54a8e098-a1f5-4b6a-9e43-0b056bb3e2e6
108 1795180332.930043 warmup 201 0.011388063430786133 cee81cf842a544e2b1700c6d5e5f4cbd 731cf770-d0eb-4aec-8aed-f5949290a301
109 1795180333.172213 warmup 201 0.01586604118347168 bf8d441585fd49b788c92dbfd279d25c 47cafb93-d453-47e7-85d2-ead4ee14c814
110 1795180333.423415 warmup 201 0.014913082122802734 aee265c80d76491ca511c2dd67e5bef7 4e051465-0aed-418b-874d-2fa35311e668
111 1795180333.67177 warmup 201 0.011767864227294922 b5b423c76c1a43fb8883ea4801d396dc 42fdaa69-7e20-43a3-8b59-7a638f9621b6
112 1795180333.928659 warmup 201 0.013628959655761719 5c0752fad40d4761bfde52ee54822e58 afc00df1-4d20-412c-872c-d0d320cf3d37
113 1795180334.175783 warmup 201 0.07439613342285156 478b50c50a3f4fe59b7aa4962d110637 ad0585cc-a72f-49dd-8545-bb0935f21245
114 1795180334.428273 warmup 201 0.013356924057006836 b0ffff00bf0a4f02a7e8de3e83944f0d 1c24e631-7a77-498e-a625-8538d60f62e5
115 1795180334.672097 warmup 201 0.013913869857788086 be440f8a05d3492a9a4816f7aa2c02a0 04df633a-94bf-4348-b397-ac7b59937040
116 1795180334.929533 warmup 201 0.6379430294036865 d8e4bcffdbd24e47ad264517c65f0dfa 6963c7af-7de7-4700-89b3-4c4bd02de125
117 1795180335.4310532 warmup 201 0.21014761924743652 e8a7cc4c3dba45daba6cabedfdbaabf0 87999bae-e07d-42e8-86c0-4a457dfabf0b
118 1795180335.179592 warmup 201 0.46358299255371094 198efae63dd14b83a1f67c8256b36fb2 2d5926f4-0437-4983-80f7-93f33a3c2dda
119 1795180335.674002 warmup 201 0.05605912208557129 c30042701c1f4f1eaef7bdc88b3920f7 f8a28688-8d60-4dcb-8d19-754ec9d586d6
120 1795180335.929294 warmup 201 0.00999593734741211 38366b0cc61c42fda055656fc8b9405c e8f7ea7c-71f2-4463-bdff-d16a901bb8f5
121 1795180336.1780539 warmup 201 0.012089014053344727 b1ef2220cce54480a9236bed82c24ff5 2d1c4a5c-7368-42ee-9fd1-f84cf3207222
122 1795180336.424557 warmup 201 0.01134800910949707 0561752001b1407a9da166b663bee655 763b99c1-ded3-4502-82f3-508af1b4b232
123 1795180336.681586 warmup 201 0.012669086456298828 b2507ead5d2f41548c5513101f01c681 a70bf5b4-cacd-408c-83b8-f73d8b9c6663
124 1795180336.92322 warmup 201 0.025698184967041016 42c5d403c13243e48ab3497efd64efe1 d1f46a4a-7792-4438-878e-b172420013ae
125 1795180337.175337 warmup 201 0.025685787200927734 732a1d48efac4a8182e803fae21b14b0 f8d80429-8488-4674-9a2c-164ff01ec638
126 1795180337.4312031 warmup 201 0.012318849563598633 551a0437a6684e04858e3e9e18e2e27a 449a474d-4d08-44bb-8773-023f2ffe5ebc
127 1795180337.676717 warmup 201 0.011137008666992188 58204786011c47978fc1a737c82d2ee0 740dc698-cbfc-4352-a7d4-4eb229b12cd8
128 1795180337.928061 warmup 201 0.010760784149169922 d5beaf3bbcd441fc93b3614c408f70c7 7477d17c-61fc-42a6-8192-e361298e1050
129 1795180338.1814332 warmup 201 0.008683919906616211 8da12b3fd0d84ca2b7e9b8b8399e6689 55629cb3-8f72-47ce-8870-02d6f8fb2e11
130 1795180338.4314299 warmup 201 0.012953042984008789 dc03191b57b740c7ba16770974d28edc 8802a764-3bb3-4474-a456-aa16351ad526
131 1795180338.674549 warmup 201 0.011281728744506836 501a10a665cd41ff9ca6f1bdf261b436 4af67448-4e3b-4100-a970-6aa2c28a60b4
132 1795180338.921731 warmup 201 0.035794973373413086 9458f35691df40d6817b1cbc7884da09 a9b1b50c-3ecb-4e40-a233-4a84efd565f4
133 1795180339.176345 warmup 201 0.01884603500366211 2424406ec5d043259e8ac310ab765f6e 0bdea318-c103-4d5e-b476-274c4feaef9b
134 1795180339.428579 warmup 201 0.011592864990234375 50aaf29486064b24b9bc50e652f46cb9 edc06291-9fb4-42bb-a501-d6c1bf6f09a0
135 1795180339.680001 warmup 201 0.009851932525634766 c6dce8e46aac45b8aefdefee3ef007f1 5a335b34-fef6-44bf-bda2-6a1b5482d587
136 1795180339.931456 warmup 201 0.010718822479248047 8b8b563bf97a4a9e9eb24d1e3f1585f2 ff857b95-1b74-4f86-b405-f7314690b2d8
137 1795180340.1814861 warmup 201 0.011487007141113281 7187e721be584781a50e5c622995bec9 f0a80079-1e90-4407-a747-a00800379415
138 1795180340.4302 warmup 201 0.019636869430541992 b91a363608904d27bd4d5f4cd801b2f3 b658efea-ec64-41d6-b268-73810fc09974
139 1795180340.671793 warmup 201 0.015497922897338867 cb3272c4ea774827b839828948ca594a be030a11-9ee2-4ee3-9f20-e0e124351756
140 1795180340.9280891 warmup 201 0.010939836502075195 c3f948a801ce48e1be201fb0a71efbbc 9190bf06-a491-489d-83e3-c12a518ad08c
141 1795180341.1780741 warmup 201 0.013393878936767578 57c094a056fb4901bcdbd9f98cb84035 64f9cb7e-3bec-42bd-976f-22f463257144
142 1795180341.423362 warmup 201 0.01970696449279785 f45ebf0ce50c4728a612832e45620dfa 82024df4-278e-4726-84e1-ed4f4c5ab451
143 1795180341.6748412 warmup 201 0.008790969848632812 ae5a9f1e20fa4882b0190f619832f229 123057d3-1867-4be0-97f9-81638814ae9e
144 1795180341.92869 warmup 201 0.08092522621154785 cf7231989bff434eb919c723499df91b f68c3ae0-ca25-4b9d-ad9d-d8ab866a052d
145 1795180342.175219 warmup 201 0.011922836303710938 095db27bed494b6780624dcd0ab805cb 847c10dc-bcc6-42ae-b2a6-c1cf38aad0ff
146 1795180342.424076 warmup 201 0.011336803436279297 9db82bcc7a2349e6a028a579b59f3731 2ac0cbf5-c8a5-4dfc-9f00-9e53cfe74ba1
147 1795180342.671635 warmup 201 0.012029886245727539 4b4531114e8842118cb57552bfe8580b c48917dd-b068-4a03-b8f8-c8ba300ea444
148 1795180342.930646 warmup 201 0.03807401657104492 77c386ecd5ad42138dbf01ed9a1a08e3 530fe2ca-019f-4498-b28a-1b46003b42fd
149 1795180343.173782 warmup 201 0.02783679962158203 9fa8bd612584480b8b8d54f9d97f84c6 71a2ea1a-a408-4b76-813c-6cc112518100
150 1795180343.431467 warmup 201 0.015299081802368164 ea11b713e9a34a16aae02e94ad918c04 8a705499-b818-4059-bf44-c742cc33385d
151 1795180343.681278 warmup 201 0.014876127243041992 06e399d5a290466fab9e5d633b60dd0a 4dcb497e-4b72-4523-9898-a2700a14d355
152 1795180343.923985 warmup 201 0.01096200942993164 cc43e56b4a2b4c19b2b6101a50c2bc94 48490016-195a-4a81-8f04-ba808859df9c
153 1795180344.171476 warmup 201 0.03735709190368652 e39608915fcb42debbe456471d2da56d 6fbc4730-e650-470b-bfc9-ffc4617929af
154 1795180344.4280179 warmup 201 0.020684003829956055 cef374c710c2463281482742a9d4f9d5 562998c0-4619-476d-9d87-0205e598889a
155 1795180344.6780381 warmup 201 0.06757974624633789 3699dadfb0c146679e5b1d18cf0ab47d 0ffc8be8-a8e3-42c4-8315-c078810a2ff7
156 1795180344.930443 warmup 201 0.01471090316772461 97b7b81f51994c4e9e513b189a6ffd37 a7a27dc8-522e-4567-961c-0db57fdea6db
157 1795180345.173086 warmup 201 0.015903949737548828 68b24d7b4e6e4d16ae9506091f733fb3 c63ccb84-a500-423f-861a-6de057114f5b
158 1795180345.42819 warmup 201 0.01735401153564453 31f7ff52721e4e61aa77a46238e37f5d 0a9ec936-7f43-45fb-8b0f-b66d7f4a7476
159 1795180345.676356 warmup 201 0.02088189125061035 d1f916e4f1934981a8fd1c3884230435 5ebfe76f-62a3-4ed8-ab48-710d781cf530
160 1795180345.92798 warmup 201 0.012233972549438477 dda6e0e723374b63a94ec455b1842d6e aa9b085d-60fc-474b-a417-e6da81114221
161 1795180346.174879 warmup 201 0.020541906356811523 00d03e3aab9e43efb45b1b99a77dd1e5 62f165a2-06ff-4919-862c-463502cd33eb
162 1795180346.4241002 warmup 201 0.01257777214050293 a9631cd4218448598816db46e05ac210 1a359611-1102-4cd0-a8a8-19f739c2d5f6
163 1795180346.6805272 warmup 201 0.031142711639404297 d6326c50cf6a4db4a14a959f047aec67 229641cd-c43f-42bb-bceb-a636fcc2e025
164 1795180346.923172 warmup 201 0.013450145721435547 fc9078ab07b6497da577535c92e91ceb 85107c40-5384-48dc-bf32-18b48892a006
165 1795180347.1747231 warmup 201 0.05333566665649414 07f0bbeaa6b44c61a7c351107b41b237 4ae310e9-fa62-43c6-b1d5-5330e738b353
166 1795180347.427953 warmup 201 0.014930009841918945 202c8996cce944b9a22892a908097a34 53140b2b-a88d-4fa6-899a-a17775f5dfaf
167 1795180347.680614 warmup 201 0.023784160614013672 907d1193824c42e5b30c45ba2634e043 ec60f205-beb2-4c00-9e47-1dd4c7a421fb
168 1795180347.926591 warmup 201 0.010828018188476562 6adcea95ee4c4ed1a2ece269f55dbedf 156e019b-22eb-4dd0-bd16-10af4e5d41a4
169 1795180348.173774 warmup 201 0.03443479537963867 f67d7dc601194d20af0034748e2c835d a2e2b4c1-d602-4451-990c-3480b5861430
170 1795180348.424813 warmup 201 0.025806903839111328 32f93effa4e745e6ba519405061bf598 1d99fe38-930b-4373-b8b1-c7a50e5883d4
171 1795180348.675287 warmup 201 0.019309043884277344 ae8cd26f20be4a35b3f63d7b46a8b960 d31bebc4-a124-4ae3-abc2-af41e2d5eae6
172 1795180348.923858 warmup 201 0.010105133056640625 bc0cc3806d7f4a919e5f47cda40b0cb4 19928ca1-9013-4dc2-b4d6-72ac6a1953e8
173 1795180349.177574 warmup 201 0.018340110778808594 786e7738d9c748d9809bd84ff34b95c4 4642a90b-25bd-491c-bf74-e6bb9890d5d6
174 1795180349.429085 warmup 201 0.04597902297973633 b1166b143286416284b919b7ee8beb1d 415730d0-e983-454a-9099-29502d67c8e4
175 1795180349.678473 warmup 201 0.12787294387817383 73a76ddd1f284f46af12cb353cc71ada 7197c218-1cc3-46a9-b2c0-77200ea16b4b
176 1795180349.921549 warmup 201 0.009884834289550781 d5c670be06a745ffbb46333a26e750f6 63e5afa5-b537-4bef-b6f9-02467ead6dad
177 1795180350.180325 warmup 201 0.014764785766601562 163c2435e5b84ac3a6d9925094a8896b 136194ea-2088-486c-87d4-d0b7644f17f5
178 1795180350.424948 warmup 201 0.015136957168579102 c06a3f595cf54357a11aa8e6e99ca18b 8385080b-1cc6-44c5-a89b-85b8244591de
179 1795180350.67535 warmup 201 0.01176905632019043 ae155b69f38b40efa06dc335484ce719 5d981da2-e2b0-4f58-945b-4e4f4fecd654
180 1795180350.928941 warmup 201 0.020803213119506836 27cf380a14bf461eb619d155d979b8f4 8c4a855a-39a7-4ef8-bbbd-90e03768e88c
181 1795180351.17329 warmup 201 0.018284082412719727 45247b8b0c484db4a857c5575a1da514 60f682a1-f799-4439-8f9a-082cbe158558
182 1795180351.423593 warmup 201 0.02286505699157715 140ac94fb75846ccbc27e9e4dff67afa f854fece-effa-4ab5-974b-72f86703bcac
183 1795180351.678071 warmup 201 0.024228811264038086 e1ab03f2591b4971a7f90801486f2bf2 609aa3e0-7e44-496f-b64c-bbc1f9fa99a9
184 1795180351.923866 warmup 201 0.01363992691040039 88c05ce549ef41c5a3a14c95c53bd409 365710a3-5f35-4adc-9c25-af3fe51493dd
185 1795180352.177956 warmup 201 0.03061199188232422 21889320216743caa5c1392d19903af3 43dbfb78-3ebe-407b-b7e1-63e8cb3b9b5a
186 1795180352.429983 warmup 201 0.012333154678344727 9de8eb1a06574abb826292e19b84f988 92e9a6bd-88fa-4610-b6c0-b5349308b4b0
187 1795180352.674007 warmup 201 0.01972508430480957 f0ce8192797a4645b76b7e5808f2192f 6b22227e-67c4-4a9a-a0ca-08a8ef20e693
188 1795180352.930673 warmup 201 0.014127016067504883 d3157bed5a4346baaec4b82e25019ecd 358d7a14-0c3d-45d2-be95-0630c878db02
189 1795180353.172828 warmup 201 0.016306161880493164 48bb51fe3a604a499d067faf39701e9b 6ab03add-9f65-4731-a343-d154bf2a40e9
190 1795180353.428106 warmup 201 0.021203994750976562 1583b7683f8141df8bb9883c0a1f189d d3aa43a2-e707-41c2-8f1a-e3e1e7634623
191 1795180353.677095 warmup 201 0.015395164489746094 d222f39f6a9a4d2e8f540d9b27aad707 320dfd89-7d56-42b7-b01c-cf04d8b513cd
192 1795180353.9265192 warmup 201 0.08537173271179199 6a936c0664b74374a9f2187a2ddbbf06 708acef1-ae2b-4229-b4cc-9d19a2a9c57e
193 1795180354.1799161 warmup 201 0.02597975730895996 c6bd819ca872476e94666e03b7f3992c b3877af3-258b-4c77-8172-72b0bbee5b27
194 1795180354.423289 warmup 201 0.010989904403686523 d30989f1247f4de7bfd4b9c32888d35c 09d175b4-c5be-4ded-bdea-47fee06baceb
195 1795180354.674009 warmup 201 0.07170987129211426 1d4784d0947541f897b9279180f8d08f 2aac78f9-62f4-4f09-8d2d-526b9b21c9b0
196 1795180354.924747 warmup 201 0.11100912094116211 ae9f5b183ecb4a67a1e29e88ecd685ca 837b2f21-0baa-4fc1-bd69-e53168ec033b
197 1795180355.171475 warmup 201 0.181365966796875 929df48dbefa466dad31befb9d377330 e47a3bea-d950-47df-ac26-483c12bafe44
198 1795180355.4301581 warmup 201 0.1972649097442627 275af00105a34802986c50739697c1ac 52788b1c-9221-4af9-8684-5b55a7aaf92a
199 1795180355.673067 warmup 201 0.08739495277404785 39a13ad72b4448f0a767afb20028d7fc 59aa1c2d-b9f9-4634-af5c-c1977cb58871
200 1795180355.92431 warmup 201 0.04182100296020508 a5a63d8af2ae4bedb575dc8739f3af2f f5b3aac5-9941-4e03-a70e-e6e5e522a110
201 1795180356.179839 warmup 201 0.06274628639221191 46d5eeb65b6d45ef9a81ec1369b7dd39 edbe7d82-fd87-423a-b49e-e395f740eb2c
202 1795180356.427257 warmup 201 0.014807939529418945 f2d123f7377c4b0cbd1649a3785a2b90 ce02ec3a-c8a8-4123-a1ad-cc29835e8825
203 1795180356.674825 warmup 201 0.011126041412353516 679d40c9ee6c481c81b28ce99f0e42b6 6c636c07-672e-4a2a-9058-db745c3463f0
204 1795180356.927894 warmup 201 0.018886804580688477 552981d06df34d508ebd5846bb836a83 ffd07a80-f592-4ddc-a3cc-ff2a9f197fb6
205 1795180357.1745431 warmup 201 0.009415864944458008 f79131e7537e4f0f8350b6fdcb0b96da bec88e0d-4f2e-44b4-8b37-91062206723c
206 1795180357.426614 warmup 201 0.04293084144592285 ae7be590ae294f2dac57e987ede2b4f6 df77674e-3378-4778-97d7-c89c7b2fa738
207 1795180357.680987 warmup 201 0.12665009498596191 4b4003e700d546d1a66698887fa1380c 8a809c0b-b8cb-4074-ac85-8e7a43dca87e
208 1795180357.9307442 warmup 201 0.022862672805786133 4fb7f73760a445dbac480366c1f672e5 6e063f18-7f51-4b19-8ed0-a0a7d73f95aa
209 1795180358.1806982 warmup 201 0.037358760833740234 57b36f91a50a4302b67b697e8f8f08a4 0e1ea419-593b-4ca3-ac11-723e02b24ff3
210 1795180358.423149 warmup 201 0.029945850372314453 603c36cdc55144618038a00cec0e2aec e6102a10-97ca-432b-beb3-fa9f02501717
211 1795180358.680745 warmup 201 0.01798725128173828 87ae437cafe647e29098811ddd6df769 b769fee9-17a2-49af-9d99-ac04d0201df4
212 1795180358.921466 warmup 201 0.015487909317016602 f8c239845d61458d975549317f54a085 7e568898-e1ff-436c-8c80-673f25a8db5f
213 1795180359.174216 warmup 201 0.016328811645507812 b052f222a93f4b7fbdef7bc74f89f2d2 bbf7df75-90b7-4ec7-abef-10d8ccce8c6c
214 1795180359.4299269 warmup 201 0.0165250301361084 25510d762ac3496ba3eeb873580d0df5 9287de57-3881-4e80-93fd-7d2e550cab75
215 1795180359.674637 warmup 201 0.018573760986328125 a4a36dd2e72d41c5af7dac81276b78f3 474e577f-89a4-485f-ae33-43dc9e0073b2
216 1795180359.928923 warmup 201 0.036789894104003906 c48bf0ba1acb4dfeb92a33a5f7c6cf31 780b5f8d-4b0f-4ec5-ae82-f21d1727f87a
217 1795180360.178257 warmup 201 0.2533531188964844 49c5d7b7dfc448e98f41e786cbdd2998 a0474355-8589-42f4-9094-7d8163b02b12
218 1795180360.421695 warmup 201 0.052854061126708984 ff183e56bb5c4ac18f6d6f3859cc5100 0d099a99-b8b0-467d-91b4-e608e2d8f1d4
219 1795180360.670847 warmup 201 0.06912517547607422 d40d667c826e49a8840ba9184578117c bb64c924-9ff3-43d9-b2b2-4441497f8878
220 1795180360.928828 warmup 201 0.08232808113098145 840f9a5f453249ba8c47882f57488674 8adf8c34-21aa-4000-829f-3775b3b5bcbf
221 1795180361.1724942 warmup 201 0.17403888702392578 8c867b70da9649899232fe5772fc5df3 69a8b185-8eec-46fc-9c3c-1d857076d989
222 1795180361.427998 warmup 201 0.021191835403442383 250cf8f6e9d54242abd898c562c6b5c9 67dae0d0-333d-4f9a-b909-014e5396107d
223 1795180361.678324 warmup 201 0.024866104125976562 4d702fdc4a814531844a755099df5f46 2de43ad9-4cfc-40a3-a168-069a77fbe13f
+373
View File
@@ -0,0 +1,373 @@
{
"date": "2026-11-20T13:11:41.242140+00:00",
"source_commit": "0fc0f0f6200476c7295ad40540aae70b7673b9a8",
"scope": "full30m",
"profile": {
"messages_per_second": 4,
"max_inflight": 16,
"upload_every_seconds": 5,
"image_pixels": [
320,
200
]
},
"complete": false,
"samples": [
{
"at": 1795180312.8492448,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 0.0,
"worker_busy": 0.0,
"worker_attempts": 31.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": null,
"slo_good": 37.0,
"slo_bad": 1.0,
"collector_received_spans": 8901.0,
"collector_sent_spans": 8886.0,
"collector_received_logs": 731.0,
"collector_sent_logs": 730.0,
"collector_failed_spans": null,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 244039680.0,
"app_memory": 657649664.0
},
"alerts": []
},
{
"at": 1795180322.153284,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 1.124622,
"worker_busy": 0.0,
"worker_attempts": 32.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.24803624999999968,
"slo_good": 38.0,
"slo_bad": 1.0,
"collector_received_spans": 9294.0,
"collector_sent_spans": 9264.0,
"collector_received_logs": 780.0,
"collector_sent_logs": 776.0,
"collector_failed_spans": null,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 244039680.0,
"app_memory": 442171392.0
},
"alerts": []
},
{
"at": 1795180331.758197,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 1.454031,
"worker_busy": 0.0,
"worker_attempts": 34.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.09944108981212381,
"slo_good": 40.0,
"slo_bad": 1.0,
"collector_received_spans": 9685.0,
"collector_sent_spans": 9667.0,
"collector_received_logs": 830.0,
"collector_sent_logs": 826.0,
"collector_failed_spans": null,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 244047872.0,
"app_memory": 442163200.0
},
"alerts": []
},
{
"at": 1795180340.560481,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 2.005743,
"worker_busy": 0.0,
"worker_attempts": 36.0,
"pool_used": 1.0,
"pool_waiting": 0.0,
"http_p95": 0.12987812499999915,
"slo_good": 42.0,
"slo_bad": 1.0,
"collector_received_spans": 10067.0,
"collector_sent_spans": 10055.0,
"collector_received_logs": 876.0,
"collector_sent_logs": 876.0,
"collector_failed_spans": null,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 244133888.0,
"app_memory": 442146816.0
},
"alerts": []
},
{
"at": 1795180349.6539578,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 0.831534,
"worker_busy": 0.0,
"worker_attempts": 38.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.09861986561062483,
"slo_good": 44.0,
"slo_bad": 1.0,
"collector_received_spans": 10477.0,
"collector_sent_spans": 10461.0,
"collector_received_logs": 930.0,
"collector_sent_logs": 928.0,
"collector_failed_spans": null,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 244137984.0,
"app_memory": 442433536.0
},
"alerts": []
},
{
"at": 1795180359.34921,
"phase": "warmup",
"signals": {
"api_error_ratio_1m": 0.0,
"queue_depth": 0.0,
"oldest_job_age": 1.390674,
"worker_busy": 1.0,
"worker_attempts": 39.0,
"pool_used": 0.0,
"pool_waiting": 0.0,
"http_p95": 0.09654615268700784,
"slo_good": 46.0,
"slo_bad": 1.0,
"collector_received_spans": 10867.0,
"collector_sent_spans": 10855.0,
"collector_received_logs": 977.0,
"collector_sent_logs": 977.0,
"collector_failed_spans": null,
"collector_queue": 0.0,
"collector_queue_capacity": 64.0,
"collector_memory": 244211712.0,
"app_memory": 442437632.0
},
"alerts": []
}
],
"phases": [
{
"name": "warmup",
"start": 1795180306.4255412,
"planned_seconds": 300
}
],
"container_limits": {
"api": {
"cpus": 0.5,
"mem_limit": "805306368",
"memswap_limit": "805306368",
"scale": 2
},
"broker": {
"cpus": 0.5,
"mem_limit": "268435456",
"memswap_limit": "268435456"
},
"cache": {
"cpus": 0.25,
"mem_limit": "134217728",
"memswap_limit": "134217728"
},
"controller": {
"cpus": 0.5,
"mem_limit": "201326592",
"memswap_limit": "201326592"
},
"grafana": {
"cpus": 0.5,
"mem_limit": "402653184",
"memswap_limit": "402653184"
},
"lb": {
"cpus": 0.5,
"mem_limit": "134217728",
"memswap_limit": "134217728"
},
"loki": {
"cpus": 0.5,
"mem_limit": "402653184",
"memswap_limit": "402653184"
},
"migrate": {
"cpus": 0.5,
"mem_limit": "268435456"
},
"observer": {
"cpus": 0.25,
"mem_limit": "201326592",
"memswap_limit": "201326592"
},
"otel": {
"cpus": 0.5,
"mem_limit": "268435456",
"memswap_limit": "268435456"
},
"postgres": {
"cpus": 1,
"mem_limit": "536870912",
"memswap_limit": "536870912"
},
"prometheus": {
"cpus": 0.5,
"mem_limit": "402653184",
"memswap_limit": "402653184"
},
"reconciler": {
"cpus": 0.25,
"mem_limit": "268435456",
"memswap_limit": "268435456"
},
"relay": {
"cpus": 0.25,
"mem_limit": "201326592",
"memswap_limit": "201326592"
},
"s3": {
"cpus": 1,
"mem_limit": "1073741824",
"memswap_limit": "1073741824"
},
"s3-bootstrap": {
"cpus": 0.25,
"mem_limit": "134217728"
},
"telemetry-init": {
"cpus": 0.1,
"mem_limit": "67108864"
},
"tempo": {
"cpus": 0.5,
"mem_limit": "536870912",
"memswap_limit": "536870912"
},
"worker": {
"cpus": 2,
"mem_limit": "1610612736",
"memswap_limit": "1610612736"
}
},
"docker_vm": {
"cpus": 8,
"memory_bytes": 6209978368
},
"runtime_versions": [
"752a5e6c0d33c2856f7ac8e66074e4adfb9a699d"
],
"elapsed_seconds": 60.619,
"requests_csv": "slo-research-final-requests.csv",
"request_statuses": {
"201": 222
},
"uploads": [
[
1795180306.424507,
"warmup",
202,
"2673e3f4-4b94-4765-a221-82a19932f481",
"9b97204e2328468687d4696df121341f"
],
[
1795180311.425008,
"warmup",
202,
"70330a66-051f-46b2-af95-ba3676a59c24",
"0f7d5b83e7c546dea644259dfa669c48"
],
[
1795180316.428648,
"warmup",
202,
"5ba1e41c-a3c0-4e3a-8773-8d149259cb0c",
"5e4502be41b549e6ab5573395740c823"
],
[
1795180321.436966,
"warmup",
202,
"16818c1b-cdd1-4a85-ac31-ebd86187e72d",
"afdbfb18874b48c1a0ceeacbda2deebf"
],
[
1795180326.444726,
"warmup",
202,
"4ded2fa3-4426-4e5d-9f83-f36050749600",
"ff3acf4079b342d087ad72006c4f8769"
],
[
1795180331.447099,
"warmup",
202,
"0e5adf2b-4b27-4efd-a0e0-46f81733df47",
"daeebdbd18e24af58d20a629d2b4119b"
],
[
1795180336.4565601,
"warmup",
202,
"e0c53652-5dc1-44f9-ad0d-35863f5af1c9",
"c2d706ff5e3d488a91d008644719b015"
],
[
1795180341.461115,
"warmup",
202,
"82d2a98d-69a6-4148-83de-81c85fc3f2c1",
"cea48c7bc3d7422998ede64dbd5f6106"
],
[
1795180346.469613,
"warmup",
202,
"28b5f6b2-a7b9-428c-9091-2a001de0dfca",
"2e52c5b0c8b84eca9c10f1955a72a20f"
],
[
1795180351.475091,
"warmup",
202,
"ec5287c6-2bd1-4de7-9ea4-ed4a4bb94026",
"ec278024a9c24ad193b8ff500bb0bfd4"
],
[
1795180356.482144,
"warmup",
202,
"1efafb25-a1bd-469f-bebc-e3fac287b86b",
"cbacdf851ac74a5c9fbed2c8c78739f0"
],
[
1795180361.482274,
"warmup",
202,
"d951a0c4-78d0-4a2c-8bfa-4880fb0bd9f6",
"c5675965b42e4644a1112505c3a83816"
]
]
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+107
View File
@@ -0,0 +1,107 @@
// Fetch the immutable module ZIP in bounded HTTP ranges. The subsequent
// go mod download validates its h1 checksum through the Go checksum database.
package main
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"time"
)
func main() {
const version = "v0.0.0-20251015172955-9e49d5e7a648"
const total int64 = 25342606
const chunk int64 = 512 * 1024
const source = "https://proxy.golang.org/github.com/minio/minio/@v/" + version + ".zip"
directory := "/go/pkg/mod/cache/download/github.com/minio/minio/@v"
if err := os.MkdirAll(directory, 0755); err != nil {
panic(err)
}
target := filepath.Join(directory, version+".zip")
if info, err := os.Stat(target); err == nil && info.Size() == total {
fmt.Println("Source ZIP cached; Go will validate checksum")
return
}
client := &http.Client{Timeout: 90 * time.Second}
jobs := make(chan int64)
errors := make(chan error, 64)
var group sync.WaitGroup
for worker := 0; worker < 8; worker++ {
group.Add(1)
go func() {
defer group.Done()
for start := range jobs {
end := min(start+chunk, total) - 1
part := fmt.Sprintf("%s.part-%d", target, start)
if info, err := os.Stat(part); err == nil && info.Size() == end-start+1 {
continue
}
var last error
for attempt := 0; attempt < 4; attempt++ {
req, _ := http.NewRequest("GET", source, nil)
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
response, err := client.Do(req)
if err != nil {
last = fmt.Errorf("source range request failed")
continue
}
expected := fmt.Sprintf("bytes %d-%d/%d", start, end, total)
if response.StatusCode != 206 || response.Header.Get("Content-Range") != expected {
response.Body.Close()
last = fmt.Errorf("unexpected source range response: HTTP %d", response.StatusCode)
continue
}
data, err := io.ReadAll(io.LimitReader(response.Body, end-start+2))
response.Body.Close()
if err != nil || int64(len(data)) != end-start+1 {
last = fmt.Errorf("incomplete source range")
continue
}
if err := os.WriteFile(part, data, 0644); err != nil {
last = err
continue
}
last = nil
fmt.Printf("Source part %d verified\n", start/chunk+1)
break
}
if last != nil {
errors <- last
}
}
}()
}
for start := int64(0); start < total; start += chunk {
jobs <- start
}
close(jobs)
group.Wait()
close(errors)
for err := range errors {
panic(err)
}
out, err := os.Create(target + ".partial")
if err != nil {
panic(err)
}
for start := int64(0); start < total; start += chunk {
part, err := os.Open(fmt.Sprintf("%s.part-%d", target, start))
if err != nil {
panic(err)
}
if _, err := io.Copy(out, part); err != nil {
panic(err)
}
part.Close()
}
if err := out.Close(); err != nil {
panic(err)
}
if err := os.Rename(target+".partial", target); err != nil {
panic(err)
}
}
+789
View File
@@ -0,0 +1,789 @@
{
"uid": "messenger-lab6",
"title": "Мессенджер: диагностика и SLO",
"schemaVersion": 41,
"version": 1,
"editable": false,
"timezone": "utc",
"refresh": "5s",
"time": {
"from": "now-30m",
"to": "now"
},
"tags": [
"lab6",
"slo"
],
"panels": [
{
"id": 1,
"title": "Запросы через балансировщик, в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\"}[1m]))",
"refId": "A",
"legendFormat": "все бизнес-запросы",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Health requests use a separate backend and are excluded. Includes ingress 503 with no API available.",
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 2,
"title": "Доля ответов 5xx",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\",code=\"5xx\"}[1m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\"}[1m])),0.001)",
"refId": "A",
"legendFormat": "ошибки 5xx",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 3,
"title": "API: p50 / p95 / p99, секунды",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.5, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
"refId": "A",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
"refId": "B",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.99, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
"refId": "C",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Quantiles from combined histogram buckets; complete ASGI response, no average of replica quantiles.",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 4,
"title": "Очередь: ожидающие задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max(messenger_queue_depth)",
"refId": "A",
"legendFormat": "ожидают",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Logical PostgreSQL jobs in queued state, not count of duplicate stream notifications.",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 5,
"title": "Возраст старейшего незавершённого задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max(messenger_oldest_job_age_seconds)",
"refId": "A",
"legendFormat": "возраст",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 6,
"title": "Обработчик: занято / доступно",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(messenger_worker_busy)",
"refId": "A",
"legendFormat": "занято",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "sum(messenger_worker_capacity)",
"refId": "B",
"legendFormat": "всего мест",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 7,
"title": "Соединения PostgreSQL",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(service_name) (messenger_db_pool_used)",
"refId": "A",
"legendFormat": "{{service_name}} занято",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "sum by(service_name) (messenger_db_pool_waiting)",
"refId": "B",
"legendFormat": "{{service_name}} ожидают",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 8,
"title": "Попытки обработки по результату",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(outcome) (rate(messenger_job_attempts_total[2m]))",
"refId": "A",
"legendFormat": "{{outcome}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 9,
"title": "Длительность попытки обработки: p95",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_duration_seconds_bucket[2m])))",
"refId": "A",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 10,
"title": "От приёма до ready: p50 / p95 / p99",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.5,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
"refId": "A",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
"refId": "B",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
"refId": "C",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Only completed publications have a duration. Deadline SLI below also counts unfinished tasks.",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 11,
"title": "SLO API за 30 минут: цель 99%",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "1 - sum(increase(haproxy_backend_http_responses_total{proxy=\"messenger\",code=\"5xx\"}[30m])) / clamp_min(sum(increase(haproxy_backend_http_responses_total{proxy=\"messenger\"}[30m])),1)",
"refId": "A",
"legendFormat": "доступность",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 12,
"title": "SLO ready за 30 минут: 99% не позже 10 секунд",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(increase(messenger_slo_decisions_total{outcome=\"good\"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)",
"refId": "A",
"legendFormat": "доля вовремя",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Durable one-time decisions per non-obsolete logical job; unfinished jobs become bad at ten seconds. Synthetic 320x200 profile.",
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 13,
"title": "CPU процессов, ядра",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(service_name)(rate(messenger_process_cpu_seconds_total[1m]))",
"refId": "A",
"legendFormat": "{{service_name}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 14,
"title": "Память процессов",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max by(service_name)(messenger_process_memory_bytes)",
"refId": "A",
"legendFormat": "{{service_name}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 15,
"title": "Буфер экспорта Collector",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "otelcol_exporter_queue_size",
"refId": "A",
"legendFormat": "{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "otelcol_exporter_queue_capacity",
"refId": "B",
"legendFormat": "{{exporter}} capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 16,
"title": "Ошибки отправки трасс и логов",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(otelcol_exporter_send_failed_spans[1m]))",
"refId": "A",
"legendFormat": "spans/s",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "sum(rate(otelcol_exporter_send_failed_log_records[1m]))",
"refId": "B",
"legendFormat": "logs/s",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 17,
"title": "Состояния локальных alerts",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "ALERTS{alertstate=\"firing\"}",
"refId": "A",
"legendFormat": "{{alertname}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 18,
"title": "Связанные логи API и обработчика",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"x": 12,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
"queryType": "range",
"datasource": {
"type": "loki",
"uid": "loki"
}
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"sortOrder": "Descending"
},
"description": "Filter structured metadata trace_id in Explore; open the same ID in the Traces datasource."
}
],
"annotations": {
"list": []
}
}
@@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: Messenger
orgId: 1
folder: Laboratory 6
type: file
disableDeletion: true
editable: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,37 @@
apiVersion: 1
datasources:
- name: Metrics
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
httpMethod: POST
timeInterval: 5s
- name: Logs
uid: loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- name: TraceID
matcherType: label
matcherRegex: trace_id
url: $${__value.raw}
datasourceUid: tempo
- name: Traces
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
filterByTraceID: true
spanStartTimeShift: -1m
spanEndTimeShift: 1m
+50
View File
@@ -0,0 +1,50 @@
global
log stdout format raw local0
maxconn 512
stats socket /tmp/admin.sock mode 600 level admin
defaults
mode http
log global
log-format "http status=%ST duration_ms=%Ta"
timeout connect 1s
timeout client 15s
timeout server 8s
timeout http-request 10s
timeout http-keep-alive 5s
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8080
use_backend health if { path_beg /health/ }
default_backend messenger
backend messenger
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 250ms fall 2 rise 1 resolvers docker init-addr libc,none resolve-prefer ipv4
server-template api 1-8 api:8080
backend health
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fall 2 rise 1 resolvers docker init-addr libc,none
server-template api 1-8 api:8080
frontend internal_metrics
bind :8404
http-request use-service prometheus-exporter if { path /metrics }
http-request deny
+6 -4
View File
@@ -1,7 +1,9 @@
# Конфигурация наблюдаемости — реализовать
# Конфигурация наблюдаемости
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
`collector.yaml`: OTLP/HTTP4318 внутри Compose, memory limiter/batch, три pipelines. Prometheus exporter8889 для приложения, self metrics8888. Log pipeline отправляется в Loki native OTLP, trace pipeline — в Tempo. Конечные очереди и повторные отправки сохраняют ограничение RAM при отказах. Эти ports на host не публикуются.
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
`prometheus.yaml`: scrape/evaluation5с, приложение/Collector/HAProxy/Tempo/Loki. `alerts.yaml`: API/ready multi-window burn и предупреждения о5xx/возрасте очереди. [RUNBOOK](RUNBOOK.md) содержит шаги диагностики и определения SLO.
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
`loki.yaml` и `tempo.yaml`: локальные persistent storage/WAL, provisioning Grafana находится в `infra/grafana`. Dashboard JSON импортируется автоматически, uid messenger-lab6,18панелей/3datasources. Версии/digests закреплены в Compose.
Результаты: [REPORT](../../REPORT.md), causal/privacy/Grafana proof `evidence/telemetry-first.json`, полный30минутный опыт `evidence/slo-research-verified.json`, анализ/график `evidence/slo-analysis-verified/`. Неполные предыдущие опыты сохранены с complete=false и не заменяют успешный результат. Отдельный clean-clone receipt должен подтвердить перезапуск и сохранение истории каждого backend.
+50
View File
@@ -0,0 +1,50 @@
# Диагностика по сохранённой телеметрии
Открой dashboard «Мессенджер: диагностика и SLO», временная зона UTC.
Сопоставляй время оповещения с метриками, трассами и связанными логами.
## API errors
Признак: ApiHighErrorRate, ApiFastBurn или ApiSlowBurn. Числитель доступности:
`sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m]))`.
Знаменатель — тот же backend со всеми code. Health использует другой backend.
Сравни API histogram, DB pool used/waiting и worker busy. В Traces найди
ошибочный server span и вложенный db.postgresql или s3.*. Длинная DB-операция
с ошибкой и занятым пулом при нормальной очереди указывает на БД. Отсутствие
API span при ingress503 может означать отсутствие готовых API. Один5xx не
доказывает причину. Через trace_id найди соответствующий Logs record.
После исправления проверь новые сообщения, спад5xx, resolved и прежние данные.
## Queue age
Признак: QueueLagGrowing, ReadyFastBurn или ReadySlowBurn. Сравни логические
queued jobs PG, oldest job age и worker busy/capacity. Upload202 при росте
ожидающей очереди и отсутствии новых worker spans указывает на обработчик.
Если попытки идут и содержат DB/S3 ошибки, причина может быть в зависимости.
Найди путь upload→producer→consumer→S3/DB. Histogram ready содержит только
завершения, поэтому отдельно проверь deadline SLI для незавершённых jobs.
После восстановления проверь тот же attachment_id, original, спад и resolved.
## Telemetry backend
Проверь backend health, Collector queue и accepted/sent/failed счётчики.
Failed attempts не равны числу уникально потерянных spans: пакет повторяется.
После слива очередей сравни изменение accepted-minus-sent и исходное состояние.
SDK queues traces/logs512records, timeout0.75s; Collector memory limiter192MiB,
batch≤256, очереди logs/traces по32пакета, один sender и finite retry≤10s.
При отказе telemetry может теряться; измеряй память и настоящие бизнес-запросы.
После восстановления найди новую трассу и лог, а не только старый screenshot.
## SLO и окна
Учебный профиль: синтетические PNG320×200, upload каждые5s,
закреплённые CPU/RAM. API цель99%ответов без5xx за30m на ingress. Jobs цель99%
ready≤10s от PG created_at принимающей транзакции до committed_at публикации.
Начальная граница консервативно включает завершение принимающей транзакции.
Однократные durable good/bad решения учитывают failed и незавершённые после10s.
Ручной replay не стирает прежний bad; устаревшие до решения jobs исключены.
Burn rate = доля плохих событий/0.01. Быстрое правило: burn>14.4 в30s и2m,
непрерывно45s. Длительное: burn>6 в2m и5m, непрерывно60s. Короткие окна выбраны
для воспроизводимого стенда, а не универсальной production paging policy.
Измеряются краткий всплеск/длительная деградация/firing/resolved/ложные события.
Точность времени ограничена scrape/evaluation5s, экспортом и реальным polling: в полном опыте медиана9.663s, максимум38.223s.
+53
View File
@@ -0,0 +1,53 @@
groups:
- name: messenger-slo
interval: 5s
rules:
- alert: ApiHighErrorRate
expr: sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) > 0.05
for: 20s
labels: {severity: warning, signal: api}
annotations:
summary: More than 5 percent business responses are 5xx
runbook: infra/otel/RUNBOOK.md#api-errors
- alert: QueueLagGrowing
expr: max(messenger_oldest_job_age_seconds) > 20 and max(messenger_queue_depth) > 0
for: 10s
labels: {severity: warning, signal: ready}
annotations:
summary: Oldest unfinished attachment exceeds 20 seconds
runbook: infra/otel/RUNBOOK.md#queue-age
- alert: ApiFastBurn
expr: (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[30s])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[30s])),0.001) / 0.01 > 14.4) and (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) / 0.01 > 14.4)
for: 45s
labels: {severity: critical, signal: api, burn: fast}
annotations:
summary: API burns the one-percent error budget in both fast windows
runbook: infra/otel/RUNBOOK.md#api-errors
- alert: ApiSlowBurn
expr: (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) / 0.01 > 6) and (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[5m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[5m])),0.001) / 0.01 > 6)
for: 60s
labels: {severity: critical, signal: api, burn: slow}
annotations:
summary: API burns the error budget across two longer windows
runbook: infra/otel/RUNBOOK.md#api-errors
- alert: ReadyFastBurn
expr: (sum(rate(messenger_slo_decisions_total{outcome="bad"}[30s])) / clamp_min(sum(rate(messenger_slo_decisions_total[30s])),0.001) / 0.01 > 14.4) and (sum(rate(messenger_slo_decisions_total{outcome="bad"}[2m])) / clamp_min(sum(rate(messenger_slo_decisions_total[2m])),0.001) / 0.01 > 14.4)
for: 45s
labels: {severity: critical, signal: ready, burn: fast}
annotations:
summary: Attachment deadline SLO burns the error budget
runbook: infra/otel/RUNBOOK.md#queue-age
- alert: ReadySlowBurn
expr: (sum(rate(messenger_slo_decisions_total{outcome="bad"}[2m])) / clamp_min(sum(rate(messenger_slo_decisions_total[2m])),0.001) / 0.01 > 6) and (sum(rate(messenger_slo_decisions_total{outcome="bad"}[5m])) / clamp_min(sum(rate(messenger_slo_decisions_total[5m])),0.001) / 0.01 > 6)
for: 60s
labels: {severity: critical, signal: ready, burn: slow}
annotations:
summary: Sustained misses of the attachment ten-second deadline
runbook: infra/otel/RUNBOOK.md#queue-age
- alert: CollectorUnavailable
expr: up{job="collector"} == 0
for: 15s
labels: {severity: warning, signal: telemetry}
annotations:
summary: Collector self metrics are unavailable
runbook: infra/otel/RUNBOOK.md#telemetry-backend
+88
View File
@@ -0,0 +1,88 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 0.0.0.0:8889
metric_expiration: 15s
resource_constant_labels:
included:
- service.name
- service.instance.id
- service.version
- deployment.environment.name
otlp_http/tempo:
endpoint: http://tempo:4318
timeout: 2s
sending_queue:
queue_size: 32
num_consumers: 1
block_on_overflow: false
retry_on_failure:
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 10s
otlp_http/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue:
queue_size: 32
num_consumers: 1
block_on_overflow: false
retry_on_failure:
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 10s
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions:
- health_check
telemetry:
logs:
level: warn
metrics:
level: detailed
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
traces:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- otlp_http/tempo
logs:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- otlp_http/loki
metrics:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- prometheus
+38
View File
@@ -0,0 +1,38 @@
auth_enabled: false
server:
http_listen_port: 3100
log_level: warn
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2026-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
ingestion_rate_mb: 4
ingestion_burst_size_mb: 8
volume_enabled: true
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes: [service.name]
- action: structured_metadata
attributes: [service.instance.id, service.version, deployment.environment.name]
analytics:
reporting_enabled: false
+23
View File
@@ -0,0 +1,23 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files: [/etc/prometheus/alerts.yaml]
scrape_configs:
- job_name: application_otlp
static_configs:
- targets: [otel:8889]
- job_name: collector
static_configs:
- targets: [otel:8888]
- job_name: ingress
static_configs:
- targets: [lb:8404]
- job_name: prometheus
static_configs:
- targets: [localhost:9090]
- job_name: tempo
static_configs:
- targets: [tempo:3200]
- job_name: loki
static_configs:
- targets: [loki:3100]
+20
View File
@@ -0,0 +1,20 @@
target: all
stream_over_http_enabled: true
server:
http_listen_port: 3200
log_level: warn
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+7
View File
@@ -0,0 +1,7 @@
{
"otel": "otel/opentelemetry-collector-contrib:0.165.0@sha256:e40d61f10c9a617cca1ffd1be4d7e2b3a4b0f0001798ff07e58cabe82e618cea",
"prometheus": "prom/prometheus:v3.16.0@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06",
"loki": "grafana/loki:3.7.10@sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7",
"tempo": "grafana/tempo:3.1.1@sha256:64e2fbd56d9c2b3412a4b430a58ea59e91a670b1e75f920a0c702213ad145044",
"grafana": "grafana/grafana:13.3.1@sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf"
}
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
root_user=$(cat /run/secrets/s3_root_user)
root_password=$(cat /run/secrets/s3_root_password)
api_key=$(cat /run/secrets/s3_access_key)
api_secret=$(cat /run/secrets/s3_secret_key)
# Credentials are passed only inside this isolated bootstrap container.
mc alias set local http://s3:9000 "$root_user" "$root_password" >/dev/null 2>&1
mc mb --ignore-existing local/messenger >/dev/null 2>&1
mc anonymous set none local/messenger >/dev/null 2>&1
mc admin user add local "$api_key" "$api_secret" >/dev/null 2>&1
mc admin policy create local messenger-api /policy.json >/dev/null 2>&1
mc admin policy attach local messenger-api --user "$api_key" >/dev/null 2>&1
printf '%s\n' 'Private bucket and scoped application policy ready'
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
export MINIO_BROWSER_REDIRECT_URL=http://127.0.0.1:9001
exec minio server /data --address :9000 --console-address 127.0.0.1:9001
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger/objects/*"]
}
]
}
+49
View File
@@ -0,0 +1,49 @@
FROM golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 AS toolchain
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOFLAGS=-p=2 GOTOOLCHAIN=local
FROM toolchain AS minio-build
WORKDIR /source
COPY infra/fetch-source.go /tmp/fetch-source.go
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go run /tmp/fetch-source.go && \
for attempt in 1 2 3 4; do \
go mod download -json github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648 && break; \
test "$attempt" -lt 4 || exit 1; sleep 1; \
done && \
cp -a /go/pkg/mod/github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648/. /source/
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
for attempt in 1 2 3; do \
timeout 900s go build -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' -o /out/minio . && break; \
test "$attempt" -lt 3 || exit 1; sleep 1; \
done
FROM toolchain AS mc-build
WORKDIR /source
RUN git init . && git remote add origin https://github.com/minio/mc.git && \
git fetch --depth 1 origin 7394ce0dd2a80935aded936b09fa12cbb3cb8096 && git checkout --detach FETCH_HEAD
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go build -trimpath -ldflags='-s -w' -o /out/mc .
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS server
LABEL org.opencontainers.image.revision=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z
COPY --from=minio-build /out/minio /usr/local/bin/minio
COPY --from=minio-build /source/LICENSE /licenses/minio.LICENSE
COPY --chmod=755 infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint
RUN mkdir /data && chown 10001:10001 /data
USER 10001:10001
ENTRYPOINT ["/usr/local/bin/s3-entrypoint"]
EXPOSE 9000
HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=30 CMD ["python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready',timeout=1).read()"]
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS bootstrap
LABEL org.opencontainers.image.revision=7394ce0dd2a80935aded936b09fa12cbb3cb8096 org.opencontainers.image.version=RELEASE.2025-08-13T08-35-41Z
COPY --from=mc-build /out/mc /usr/local/bin/mc
COPY --from=mc-build /source/LICENSE /licenses/mc.LICENSE
COPY --chmod=755 infra/s3-bootstrap.sh /usr/local/bin/s3-bootstrap
COPY infra/s3-policy.json /policy.json
USER 10001:10001
ENV MC_CONFIG_DIR=/tmp/mc
ENTRYPOINT ["/usr/local/bin/s3-bootstrap"]
+36
View File
@@ -0,0 +1,36 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL
);
CREATE TABLE members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
position integer NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX members_user_chat_idx ON members(user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL,
FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
CREATE TABLE sessions (
token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32),
user_id uuid NOT NULL REFERENCES users(id),
created_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE idempotency (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
body_hash bytea NOT NULL CHECK (octet_length(body_hash)=32),
message_id uuid NOT NULL REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY(user_id, chat_id, key)
);
CREATE INDEX idempotency_expiry_idx ON idempotency(expires_at);
+32
View File
@@ -0,0 +1,32 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
ALTER TABLE messages ADD UNIQUE(chat_id,id);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL,
filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
state text NOT NULL CHECK(state IN ('pending','ready','failed')),
error_code text,
uploader_instance text NOT NULL,
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(chat_id,id),
FOREIGN KEY(chat_id,owner_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX attachments_state_created_idx ON attachments(state,created_at,id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position integer NOT NULL CHECK(position BETWEEN 0 AND 9),
PRIMARY KEY(message_id,attachment_id),
UNIQUE(message_id,position),
FOREIGN KEY(chat_id,message_id) REFERENCES messages(chat_id,id),
FOREIGN KEY(chat_id,attachment_id) REFERENCES attachments(chat_id,id)
);
+54
View File
@@ -0,0 +1,54 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
ALTER TABLE attachments ADD CHECK(state IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>=1);
ALTER TABLE attachments ADD COLUMN published_version integer;
CREATE TABLE upload_reservations (
id uuid PRIMARY KEY, object_key text UNIQUE NOT NULL,
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds'
);
CREATE TABLE jobs (
id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL CHECK(pipeline_version>=1), source_key text NOT NULL,
state text NOT NULL CHECK(state IN ('queued','processing','ready','failed','obsolete')),
attempt integer NOT NULL DEFAULT 0, cycle integer NOT NULL DEFAULT 1,
fence bigint NOT NULL DEFAULT 0, lease_owner text, lease_until timestamptz,
next_at timestamptz NOT NULL DEFAULT clock_timestamp(), error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(attachment_id,pipeline_version)
);
CREATE INDEX jobs_due_idx ON jobs(state,next_at);
CREATE TABLE outbox (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id),
payload jsonb NOT NULL, due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz
);
CREATE INDEX outbox_due_idx ON outbox(due_at,id) WHERE published_at IS NULL;
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
name text NOT NULL CHECK(name IN ('original','thumbnail','crop','optimized')),
object_key text UNIQUE NOT NULL, content_type text NOT NULL,
size_bytes bigint NOT NULL, sha256 text NOT NULL,
width integer, height integer,
PRIMARY KEY(attachment_id,pipeline_version,name)
);
CREATE TABLE effect_publications (
attachment_id uuid NOT NULL, pipeline_version integer NOT NULL,
job_id uuid NOT NULL REFERENCES jobs(id), fence bigint NOT NULL,
committed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY(attachment_id,pipeline_version)
);
CREATE TABLE pipeline_audit (
id bigserial PRIMARY KEY, job_id uuid NOT NULL, action text NOT NULL,
operator text NOT NULL, reason text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
-- Fault injection is disabled unless PIPELINE_FAULTS=1; no public endpoint.
CREATE TABLE pipeline_faults (
id bigserial PRIMARY KEY, target uuid NOT NULL, phase text NOT NULL,
action text NOT NULL CHECK(action IN ('wait','crash','transient','permanent','hang')),
remaining integer NOT NULL DEFAULT 1, hit_at timestamptz, released_at timestamptz
);
CREATE TABLE pipeline_events (
id bigserial PRIMARY KEY, job_id uuid, event text NOT NULL,
detail jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
+10
View File
@@ -0,0 +1,10 @@
ALTER TABLE jobs ADD COLUMN traceparent text NOT NULL DEFAULT '';
ALTER TABLE jobs ADD COLUMN tracestate text NOT NULL DEFAULT '';
CREATE TABLE job_slo_observations (
job_id uuid PRIMARY KEY REFERENCES jobs(id),
outcome text NOT NULL CHECK(outcome IN ('good','bad')),
decided_at timestamptz NOT NULL DEFAULT clock_timestamp(),
histogram_epoch uuid
);
CREATE INDEX job_slo_epoch_idx ON job_slo_observations(histogram_epoch);
CREATE INDEX effect_publications_time_idx ON effect_publications(committed_at,job_id);
+58
View File
@@ -0,0 +1,58 @@
[project]
name = "mtusi-messenger-lab6"
version = "1.0.0"
requires-python = ">=3.14,<3.15"
dependencies = [
"fastapi==0.143.3",
"pydantic==2.14.1",
"uvicorn==0.54.0",
"psycopg[binary,pool]==3.3.6",
"argon2-cffi>=25,<26",
"redis==8.1.0",
"minio==7.2.20",
"pillow==13.0.0",
"opentelemetry-api==1.45.2",
"opentelemetry-sdk==1.45.2",
"opentelemetry-exporter-otlp-proto-http==1.45.2",
"psutil==7.1.3",
]
[dependency-groups]
dev = [
"pytest>=8,<10",
"httpx>=0.28,<1",
"ruff>=0.14,<1",
"openapi-spec-validator==0.7.2",
"psutil>=7,<8",
"matplotlib>=3.10,<4",
"pyyaml==6.0.3",
]
[tool.ruff]
line-length = 79
target-version = "py314"
include = ["src/**/*.py", "student-tests/**/*.py", "student-unit/**/*.py", "research/**/*.py", "scripts/**/*.py"]
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
[tool.ruff.per-file-target-version]
"scripts/*.py" = "py39"
# SQL и генерируемые Markdown-таблицы в инструментах не разрываем.
[tool.ruff.lint.per-file-ignores]
"research/*.py" = ["E501"]
"scripts/*.py" = ["E501"]
# Keep SQL statements together; the formatter controls executable statements.
"src/attachments.py" = ["E501"]
"src/pipeline.py" = ["E501"]
"src/worker.py" = ["E501"]
"src/operator.py" = ["E501"]
"src/reconcile.py" = ["E501"]
"src/observer.py" = ["E501"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra"
+128
View File
@@ -0,0 +1,128 @@
"""Summarize measured fault invariants and child resource samples."""
import argparse
import csv
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def effects(case):
for key in ["effect", "v1", "v2"]:
if key in case:
yield key, case[key]
for i, item in enumerate(case.get("effects", [])):
yield str(i + 1), item
def main():
p = argparse.ArgumentParser()
p.add_argument("input", type=Path)
p.add_argument("--output", required=True, type=Path)
a = p.parse_args()
d = json.loads(a.input.read_text())
assert d["complete"] and d.get("scope", "all") == "all"
rows = []
for case in d["cases"]:
for role, e in effects(case):
j = e["job"]
rows.append(
dict(
case=case["case"],
role=role,
job_id=j["id"],
version=j["pipeline_version"],
attempt=j["attempt"],
cycle=j["cycle"],
fence=j["fence"],
state=j["state"],
publications=e["publications"],
variant_rows=len(e["variants"]),
physical_keys=len(e["physical_keys"]),
)
)
a.output.mkdir(parents=True, exist_ok=True)
with (a.output / "effects.csv").open("w") as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
ready = [r for r in rows if r["state"] == "ready"]
assert all(
r["publications"] == 1
and r["variant_rows"] == 4
and r["physical_keys"] == 4
for r in ready
)
stale = [r for r in rows if r["state"] == "obsolete"]
assert all(r["publications"] == 0 for r in stale)
job_ids = {r["job_id"] for r in rows}
samples = [
x["detail"]
for x in d["worker_metrics"]
if str(x["job_id"]) in job_ids
and x["detail"].get("result") == "ready"
and "rss_peak_mib" in x["detail"]
]
stats = {
"source_commit": d["source_commit"],
"cases": len(d["cases"]),
"observed_ready_effects": len(ready),
"observed_obsolete_effects": len(stale),
"all_ready_have_one_publication_and_four_variants": True,
"all_obsolete_have_zero_publications": True,
"successful_child_samples": len(samples),
"child_elapsed_range_seconds": [
min(x["seconds"] for x in samples),
max(x["seconds"] for x in samples),
],
"child_cpu_range_seconds": [
min(x["cpu_seconds"] for x in samples),
max(x["cpu_seconds"] for x in samples),
],
"child_peak_rss_range_mib": [
min(x["rss_peak_mib"] for x in samples),
max(x["rss_peak_mib"] for x in samples),
],
"scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned",
}
(a.output / "summary.json").write_text(json.dumps(stats, indent=2) + "\n")
fig, axes = plt.subplots(1, 2, figsize=(13, 5), layout="constrained")
axes[0].bar(
range(1, len(rows) + 1),
[r["publications"] for r in rows],
color=[
"#246b8e" if r["state"] == "ready" else "#d17d20" for r in rows
],
)
axes[0].axhline(1, color="#5b6870", linestyle="--", linewidth=1)
axes[0].set(
xlabel="Effect observation (CSV row)",
ylabel="Committed publications",
title="One publication per ready version; stale v1 = 0",
yticks=[0, 1],
)
axes[1].scatter(
[x["seconds"] for x in samples],
[x["rss_peak_mib"] for x in samples],
color="#246b8e",
s=35,
alpha=0.8,
)
axes[1].set(
xlabel="Child elapsed, seconds",
ylabel="Child peak RSS, MiB",
title=f"{len(samples)} completed child samples (synthetic320×200)",
)
for ax in axes:
ax.grid(axis="y", alpha=0.2)
fig.savefig(a.output / "pipeline-results.png", dpi=150)
plt.close(fig)
print(json.dumps(stats, indent=2))
if __name__ == "__main__":
main()
+256
View File
@@ -0,0 +1,256 @@
"""Derive tables and a figure from completed observations, never simulated data."""
import argparse
import csv
import json
import math
from collections import Counter
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def quantile(values, fraction):
ordered = sorted(values)
return (
ordered[max(0, math.ceil(len(ordered) * fraction) - 1)]
if ordered
else None
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("input", type=Path)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
report = json.loads(args.input.read_text())
assert report["complete"] and report["scope"] == "full30m"
assert report["elapsed_seconds"] >= 1800
with (args.input.parent / report["requests_csv"]).open() as stream:
requests = list(csv.DictReader(stream))
samples = report["samples"]
origin = samples[0]["at"]
summaries = []
for phase in report["phases"]:
# Labels remain unchanged while orchestration waits between holds.
# Attribute writes by their actual start time inside the measured hold.
rows = [
r
for r in requests
if phase["start"] <= float(r["timestamp"]) < phase["end"]
]
successful = [
float(r["seconds"]) for r in rows if r["status"] == "201"
]
elapsed = phase["end"] - phase["start"]
summaries.append(
{
"phase": phase["name"],
"seconds": round(elapsed, 3),
"started_requests": len(rows),
"statuses": dict(Counter(r["status"] for r in rows)),
"successful_rps": round(len(successful) / elapsed, 3),
"success_p50_ms": round(quantile(successful, 0.5) * 1000, 3)
if successful
else None,
"success_p95_ms": round(quantile(successful, 0.95) * 1000, 3)
if successful
else None,
"success_p99_ms": round(quantile(successful, 0.99) * 1000, 3)
if successful
else None,
"critical_firing_names": sorted(
n
for n in phase["firing"]
if n
in {
"ApiFastBurn",
"ApiSlowBurn",
"ReadyFastBurn",
"ReadySlowBurn",
}
),
"warning_firing_names": sorted(
n
for n in phase["firing"]
if n
not in {
"ApiFastBurn",
"ApiSlowBurn",
"ReadyFastBurn",
"ReadySlowBurn",
}
),
}
)
intervals = {}
for name in sorted({a["name"] for s in samples for a in s["alerts"]}):
segments, start = [], None
for sample in samples:
active = any(
a["name"] == name and a["state"] == "firing"
for a in sample["alerts"]
)
if active and start is None:
start = sample["at"]
elif not active and start is not None:
segments.append([start, sample["at"]])
start = None
if start is not None:
segments.append([start, samples[-1]["at"]])
intervals[name] = segments
gap = [
b["at"] - a["at"] for a, b in zip(samples, samples[1:], strict=False)
]
memory = [
s["signals"]["collector_memory"] / 1024**2
for s in samples
if s["signals"]["collector_memory"] is not None
]
summary = {
"source_commit": report["source_commit"],
"runtime_versions": report["runtime_versions"],
"elapsed_seconds": report["elapsed_seconds"],
"sample_count": len(samples),
"sample_gap_seconds": {
"min": min(gap),
"median": quantile(gap, 0.5),
"max": max(gap),
},
"client_quantile_method": "nearest rank of successful end-to-end writes; not histogram interpolation",
"phase_assignment": "request start timestamp inside measured hold; orchestration gaps excluded",
"phases": summaries,
"incidents": report["incidents"],
"firing_intervals_unix_seconds": intervals,
"short_spikes": report["short_spikes"],
"collector_rss_mib": {"min": min(memory), "max": max(memory)},
"collector_max_queued_packets_sum": max(
s["signals"]["collector_queue"] or 0 for s in samples
),
"collector_queue_capacity_sum": max(
s["signals"]["collector_queue_capacity"] or 0 for s in samples
),
"backend_outage": report["backend_outage"],
"preservation": report["preservation"],
"final_30m_sli": report["final_30m_sli"],
}
args.output.mkdir(parents=True, exist_ok=True)
(args.output / "summary.json").write_text(
json.dumps(summary, ensure_ascii=False, indent=2) + "\n"
)
with (args.output / "phases.csv").open("w") as stream:
writer = csv.DictWriter(stream, fieldnames=list(summaries[0]))
writer.writeheader()
writer.writerows(summaries)
x = [(s["at"] - origin) / 60 for s in samples]
fig, axes = plt.subplots(
4, 1, figsize=(12, 12), sharex=True, layout="constrained"
)
for axis in axes:
axis.grid(alpha=0.2)
for phase in report["phases"]:
if (
phase["name"].endswith("_sustained")
or phase["name"] == "backend_unavailable"
):
color = (
"#cf704e"
if phase["name"].startswith("api")
else "#8b79b5"
if phase["name"].startswith("ready")
else "#85919d"
)
axis.axvspan(
(phase["start"] - origin) / 60,
(phase["end"] - origin) / 60,
color=color,
alpha=0.15,
)
axes[0].plot(
x,
[(s["signals"]["api_error_ratio_1m"] or 0) * 100 for s in samples],
color="#b04a32",
label="API 5xx ratio, rolling 1 min",
)
axes[0].axhline(1, color="black", ls=":", label="1% error budget")
axes[0].set_ylabel("5xx, %")
axes[0].legend(loc="upper right")
axes[0].set_title(
"Actual 30-minute SLO experiment: DB delay, stopped worker, unavailable telemetry stores"
)
axes[1].plot(
x,
[s["signals"]["queue_depth"] for s in samples],
color="#386a9b",
label="Queued jobs",
)
axes[1].set_ylabel("Jobs")
other = axes[1].twinx()
other.plot(
x,
[s["signals"]["oldest_job_age"] for s in samples],
color="#8b579d",
label="Oldest unfinished job",
)
other.axhline(20, ls=":", color="#8b579d")
other.set_ylabel("Age, seconds")
axes[1].legend(loc="upper left")
other.legend(loc="upper right")
alert_names = [
"ApiFastBurn",
"ApiSlowBurn",
"ReadyFastBurn",
"ReadySlowBurn",
]
for index, name in enumerate(alert_names):
axes[2].broken_barh(
[
((start - origin) / 60, (end - start) / 60)
for start, end in intervals.get(name, [])
],
(index - 0.3, 0.6),
facecolors="#bd523c",
)
axes[2].set_yticks(range(4), alert_names)
axes[2].set_ylim(-0.6, 3.6)
axes[2].set_ylabel("Critical alert firing")
axes[3].plot(
x,
[
s["signals"]["collector_memory"] / 1024**2
if s["signals"]["collector_memory"] is not None
else None
for s in samples
],
color="#347465",
label="Collector RSS",
)
axes[3].axhline(
report["container_limits"]["otel"]["mem_limit"] / 1024**2,
ls=":",
color="black",
label="Container memory limit",
)
axes[3].set_ylabel("MiB")
axes[3].set_xlabel("Minutes from first observation")
other = axes[3].twinx()
other.plot(
x,
[s["signals"]["collector_queue"] for s in samples],
color="#9c6e2e",
label="Queued telemetry packets, sum",
)
other.set_ylabel("Packets")
axes[3].legend(loc="upper left")
other.legend(loc="upper right")
fig.savefig(args.output / "slo-timeline.png", dpi=150)
plt.close(fig)
print("PASS: measured phase statistics, alert intervals and timeline")
if __name__ == "__main__":
main()
+64
View File
@@ -0,0 +1,64 @@
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def create(path, value):
try:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
except FileExistsError:
return
with os.fdopen(fd, "w") as stream:
stream.write(value + "\n")
def main():
directory = ROOT / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
pg = directory / "postgres_password"
cache = directory / "cache_password"
create(pg, secrets.token_urlsafe(32))
create(cache, secrets.token_urlsafe(32))
create(
directory / "database_url",
"postgresql://messenger:"
+ pg.read_text().strip()
+ "@postgres:5432/messenger",
)
create(
directory / "cache_url",
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
)
create(
directory / "valkey.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
+ cache.read_text().strip(),
)
create(directory / "s3_root_user", secrets.token_hex(10))
create(directory / "s3_root_password", secrets.token_urlsafe(32))
create(directory / "s3_access_key", secrets.token_hex(10))
create(directory / "s3_secret_key", secrets.token_urlsafe(32))
create(directory / "grafana_password", secrets.token_urlsafe(32))
create(directory / "broker_password", secrets.token_urlsafe(32))
create(
directory / "broker_url",
"redis://:"
+ (directory / "broker_password").read_text().strip()
+ "@broker:6379/0",
)
create(
directory / "broker.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass "
+ (directory / "broker_password").read_text().strip(),
)
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
print("Local configuration ready; existing files preserved")
if __name__ == "__main__":
main()
+83
View File
@@ -0,0 +1,83 @@
"""Явные операторские действия только над API текущего Compose-проекта."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
def containers():
ids = command("docker", "compose", "ps", "-q", "api").split()
return json.loads(command("docker", "inspect", *ids)) if ids else []
def alias(cid):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=2); "
"print(r.headers['X-Instance-Id'])"
)
return command("docker", "exec", cid, "python", "-c", code)
def wait_count(count):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
data = containers()
if len(data) == count and all(
c["State"].get("Health", {}).get("Status") == "healthy"
for c in data
):
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
time.sleep(3)
return
time.sleep(0.5)
raise TimeoutError("Replica readiness")
def main():
action = sys.argv[1]
data = containers()
if action == "stop":
target = os.environ["TARGET_INSTANCE_ID"]
matches = [c for c in data if alias(c["Id"]) == target]
if len(matches) != 1 or len(data) < 2:
raise RuntimeError("Target must identify one running replica")
command("docker", "stop", "--time", "10", matches[0]["Id"])
print("Selected API stopped")
elif action == "rolling":
count = len(data)
if count < 2:
raise RuntimeError("Need at least two replicas")
# Сначала добавляется готовая реплика, затем удаляется одна старая.
for old in data:
command(
"docker",
"compose",
"up",
"-d",
"--no-deps",
"--no-recreate",
"--scale",
f"api={count + 1}",
"api",
)
wait_count(count + 1)
command("docker", "stop", "--time", "10", old["Id"])
command("docker", "rm", old["Id"])
wait_count(count)
print("Sequential replacement completed")
else:
raise ValueError("Unknown action")
if __name__ == "__main__":
main()
+134
View File
@@ -0,0 +1,134 @@
"""Read-only queries through an internal probe, without exposing admin ports."""
import base64
import json
import time
from urllib.parse import urlencode
from scripts.stand import compose
ENDPOINTS = {
"prometheus": "http://prometheus:9090",
"loki": "http://loki:3100",
"tempo": "http://tempo:3200",
"otel": "http://otel:13133",
}
def fetch(service, path, *, params=None, json_result=True):
url = ENDPOINTS[service] + path
if params:
url += "?" + urlencode(params, doseq=True)
code = (
"import urllib.request; "
f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8); "
"print(r.read().decode())"
)
text = compose("exec", "-T", "observer", "python", "-c", code)
return json.loads(text) if json_result else text
def query(expression, *, at=None):
params = {"query": expression}
if at is not None:
params["time"] = str(at)
result = fetch("prometheus", "/api/v1/query", params=params)
assert result["status"] == "success"
return result["data"]["result"]
def scalar(expression):
result = query(expression)
return float(result[0]["value"][1]) if result else 0.0
def queries(expressions):
urls = {
name: ENDPOINTS["prometheus"]
+ "/api/v1/query?"
+ urlencode({"query": expression})
for name, expression in expressions.items()
}
code = (
"import json,urllib.request\n"
f"urls={urls!r}\n"
"result={}\n"
"for name,url in urls.items():\n"
" with urllib.request.urlopen(url,timeout=8) as r: result[name]=json.load(r)['data']['result']\n"
"print(json.dumps(result))"
)
return json.loads(compose("exec", "-T", "observer", "python", "-c", code))
def logs(trace_id, *, start=None):
return fetch(
"loki",
"/loki/api/v1/query_range",
params={
"query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="'
+ trace_id
+ '"',
"start": str(int((start or time.time() - 3600) * 1e9)),
"end": str(time.time_ns()),
"limit": 500,
},
)
def normalize_id(value):
if not value:
return ""
if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value):
return value
return base64.b64decode(value).hex()
def spans(trace_id):
result = fetch("tempo", "/api/traces/" + trace_id)
flattened = []
for batch in result.get("batches", result.get("resourceSpans", [])):
resource = {
a["key"]: next(iter(a["value"].values()))
for a in batch.get("resource", {}).get("attributes", [])
}
for scope in batch.get("scopeSpans", []):
for item in scope.get("spans", []):
flattened.append(
{
"service": resource.get("service.name"),
"version": resource.get("service.version"),
"instance": resource.get("service.instance.id"),
"name": item["name"],
"kind": item.get("kind"),
"trace_id": normalize_id(item.get("traceId")),
"span_id": normalize_id(item.get("spanId")),
"parent_span_id": normalize_id(
item.get("parentSpanId")
),
"duration_seconds": (
int(item["endTimeUnixNano"])
- int(item["startTimeUnixNano"])
)
/ 1e9,
"status": item.get("status", {}),
"attributes": {
a["key"]: next(iter(a["value"].values()))
for a in item.get("attributes", [])
},
}
)
return flattened
def wait(condition, *, timeout=90, interval=1):
end = time.monotonic() + timeout
last_error = None
while time.monotonic() < end:
try:
result = condition()
if result:
return result
except (RuntimeError, ValueError, KeyError) as exc:
last_error = type(exc).__name__
time.sleep(interval)
raise TimeoutError("Telemetry observation not reached: " + str(last_error))
+633
View File
@@ -0,0 +1,633 @@
"""Thirty-minute controlled SLO study with real DB/worker/backend failures."""
import argparse
import csv
import io
import json
import math
import os
import threading
import time
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
from scripts.observe import fetch, logs, queries, spans, wait
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
ERRORS = 'haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}'
RESPONSES = 'haproxy_backend_http_responses_total{proxy="messenger"}'
SIGNALS = {
"api_error_ratio_1m": f"sum(rate({ERRORS}[1m])) / clamp_min(sum(rate({RESPONSES}[1m])),0.001)",
"queue_depth": "max(messenger_queue_depth)",
"oldest_job_age": "max(messenger_oldest_job_age_seconds)",
"worker_busy": "sum(messenger_worker_busy)",
"worker_attempts": "sum(messenger_job_attempts_total)",
"pool_used": 'sum(messenger_db_pool_used{service_name="messenger-api"})',
"pool_waiting": 'sum(messenger_db_pool_waiting{service_name="messenger-api"})',
"http_p95": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))",
"slo_good": 'sum(messenger_slo_decisions_total{outcome="good"})',
"slo_bad": 'sum(messenger_slo_decisions_total{outcome="bad"})',
"collector_received_spans": "sum(otelcol_receiver_accepted_spans)",
"collector_sent_spans": "sum(otelcol_exporter_sent_spans)",
"collector_received_logs": "sum(otelcol_receiver_accepted_log_records)",
"collector_sent_logs": "sum(otelcol_exporter_sent_log_records)",
"collector_failed_spans": "sum(otelcol_exporter_send_failed_spans)",
"collector_queue": "sum(otelcol_exporter_queue_size)",
"collector_queue_capacity": "sum(otelcol_exporter_queue_capacity)",
"collector_memory": 'otelcol_process_memory_rss{job="collector"}',
"app_memory": "sum(messenger_process_memory_bytes)",
}
BURN_ALERTS = {"ApiFastBurn", "ApiSlowBurn", "ReadyFastBurn", "ReadySlowBurn"}
def snapshot(phase):
observed = queries(SIGNALS)
values = {}
for key, rows in observed.items():
value = float(rows[0]["value"][1]) if rows else None
values[key] = (
value if value is not None and math.isfinite(value) else None
)
alerts = fetch("prometheus", "/api/v1/alerts")["data"]["alerts"]
return {
"at": time.time(),
"phase": phase,
"signals": values,
"alerts": [
{
"name": a["labels"]["alertname"],
"state": a["state"],
"active_at": a.get("activeAt"),
}
for a in alerts
],
}
def firing(samples):
return {
a["name"]
for sample in samples
for a in sample["alerts"]
if a["state"] == "firing"
}
class Workload:
def __init__(self, client, headers, chat):
self.client, self.headers, self.chat = client, headers, chat
self.stop = threading.Event()
self.phase = "warmup"
self.requests = []
self.uploads = []
self.lock = threading.Lock()
self.slots = threading.BoundedSemaphore(16)
self.pool = ThreadPoolExecutor(max_workers=16)
image = io.BytesIO()
Image.new("RGB", (320, 200), (39, 102, 157)).save(image, "PNG")
self.raw = image.getvalue()
self.threads = [
threading.Thread(target=self.schedule_messages),
threading.Thread(target=self.schedule_uploads),
]
def write(self):
started, phase, trace_id = time.time(), self.phase, uuid4().hex
status, identity = 0, ""
try:
response = self.client.post(
f"/api/v1/chats/{self.chat}/messages",
headers={
**self.headers,
"Idempotency-Key": uuid4().hex,
"traceparent": "00-"
+ trace_id
+ "-"
+ uuid4().hex[:16]
+ "-01",
},
json={"text": "Synthetic SLO message " + uuid4().hex},
)
status = response.status_code
if status == 201:
identity = response.json()["id"]
except httpx.HTTPError:
pass
finally:
with self.lock:
self.requests.append(
[
started,
phase,
status,
time.time() - started,
trace_id,
identity,
]
)
self.slots.release()
def schedule_messages(self):
next_at = time.monotonic()
while not self.stop.is_set():
if self.slots.acquire(blocking=False):
self.pool.submit(self.write)
else:
with self.lock:
self.requests.append(
[time.time(), self.phase, -1, 0, "", ""]
)
next_at += 0.25
self.stop.wait(max(0, next_at - time.monotonic()))
def schedule_uploads(self):
while not self.stop.is_set():
started, phase, trace_id = time.time(), self.phase, uuid4().hex
status, identity = 0, ""
try:
response = self.client.post(
f"/api/v1/chats/{self.chat}/attachments",
params={"filename": "synthetic-slo.png"},
content=self.raw,
headers={
**self.headers,
"Content-Type": "image/png",
"traceparent": "00-"
+ trace_id
+ "-"
+ uuid4().hex[:16]
+ "-01",
},
)
status = response.status_code
if status == 202:
identity = response.json()["id"]
except httpx.HTTPError:
pass
with self.lock:
self.uploads.append(
[started, phase, status, identity, trace_id]
)
self.stop.wait(max(0, 5 - (time.time() - started)))
def start(self):
for thread in self.threads:
thread.start()
def close(self):
self.stop.set()
for thread in self.threads:
thread.join(20)
self.pool.shutdown(wait=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/slo-research-first.json")
parser.add_argument("--minimum-seconds", type=int, default=1800)
args = parser.parse_args()
output = ROOT / args.output
assert not output.exists(), "Choose a new output path"
report = {
"date": datetime.now(UTC).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"scope": "full30m"
if args.minimum_seconds >= 1800
else "short-development",
"profile": {
"messages_per_second": 4,
"max_inflight": 16,
"upload_every_seconds": 5,
"image_pixels": [320, 200],
},
"complete": False,
"samples": [],
"phases": [],
}
study_started = time.monotonic()
output.parent.mkdir(parents=True, exist_ok=True)
load = None
trigger = "lab6_slo_delay_" + uuid4().hex[:8]
with httpx.Client(
base_url=BASE,
timeout=8,
trust_env=False,
limits=httpx.Limits(max_connections=24, max_keepalive_connections=24),
) as client:
ready(timeout=90)
config = json.loads(compose("config", "--format", "json"))
report["container_limits"] = {
name: {
k: int(service[k])
if k in {"mem_limit", "memswap_limit", "scale"}
else service[k]
for k in ["cpus", "mem_limit", "memswap_limit", "scale"]
if k in service
}
for name, service in config["services"].items()
}
report["docker_vm"] = json.loads(
command(
[
"docker",
"info",
"--format",
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}',
]
)
)
report["runtime_versions"] = sorted(
{
row["metric"].get("service_version", "unknown")
for row in queries(
{"versions": "messenger_process_memory_bytes"}
)["versions"]
}
)
user, _, session = actor(client)
headers = auth(session)
response = client.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Synthetic SLO experiment",
"member_ids": [user["id"]],
},
)
assert response.status_code == 201
chat = response.json()["id"]
load = Workload(client, headers, chat)
workload = load
def save():
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
def hold(name, seconds, *, until_clear=None, max_seconds=None):
load.phase = name
phase = {
"name": name,
"start": time.time(),
"planned_seconds": seconds,
"maximum_seconds": max_seconds or seconds,
}
report["phases"].append(phase)
minimum_end = time.monotonic() + seconds
deadline = time.monotonic() + (max_seconds or seconds)
start_index = len(report["samples"])
while time.monotonic() < deadline:
sample = snapshot(name)
report["samples"].append(sample)
save()
if (
until_clear is not None
and time.monotonic() >= minimum_end
and not until_clear.intersection(firing([sample]))
):
break
time.sleep(min(5, max(0, deadline - time.monotonic())))
phase["end"] = time.time()
phase["firing"] = sorted(firing(report["samples"][start_index:]))
return phase
def db_delay(on):
if on:
sql(
f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='{chat}'::uuid THEN PERFORM pg_sleep(3); END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {trigger}()"
)
else:
sql(
f"DROP TRIGGER IF EXISTS {trigger} ON messages; DROP FUNCTION IF EXISTS {trigger}()"
)
try:
load.start()
warm = hold("warmup", 300)
settled = [
s for s in report["samples"] if s["at"] >= warm["end"] - 60
]
assert not BURN_ALERTS.intersection(firing(settled)), (
"Baseline is not clean"
)
assert (
report["samples"][-1]["signals"]["collector_memory"]
is not None
)
report["baseline"] = snapshot("baseline")
for signal in ["api", "ready"]:
if signal == "api":
activate, restore = (
lambda: db_delay(True),
lambda: db_delay(False),
)
else:
def activate():
return compose("stop", "--timeout", "1", "worker")
def restore():
return compose(
"up", "-d", "--no-deps", "--no-recreate", "worker"
)
brief_started = time.time()
activate()
brief = hold(signal + "_brief", 5)
restore()
settle = hold(
signal + "_brief_recovery", 120 if signal == "api" else 60
)
assert not BURN_ALERTS.intersection(
brief["firing"] + settle["firing"]
), "Brief incident triggered paging"
report.setdefault("short_spikes", []).append(
{
"signal": signal,
"action_started": brief_started,
"paging_events": 0,
}
)
action_started = time.time()
activate()
action_finished = time.time()
degraded = hold(
signal + "_sustained", 180 if signal == "api" else 150
)
restore_started = time.time()
restore()
restore_finished = time.time()
expected = (
{"ApiFastBurn", "ApiSlowBurn", "ApiHighErrorRate"}
if signal == "api"
else {"ReadyFastBurn", "ReadySlowBurn", "QueueLagGrowing"}
)
recovered = hold(
signal + "_recovery",
150 if signal == "api" else 240,
until_clear=expected,
max_seconds=600,
)
assert expected <= set(degraded["firing"]), (
signal,
degraded["firing"],
)
last = report["samples"][-1]
assert not expected.intersection(firing([last])), (
"Incident alerts did not resolve"
)
points = [
s
for s in report["samples"]
if action_finished <= s["at"] <= recovered["end"]
]
timings = {}
for name in sorted(expected):
first = next(
s["at"] for s in points if name in firing([s])
)
resolved = next(
s["at"]
for s in points
if s["at"] >= restore_finished
and name not in firing([s])
)
timings[name] = {
"detect_seconds": round(first - action_finished, 3),
"resolve_seconds": round(
resolved - restore_finished, 3
),
}
report.setdefault("incidents", []).append(
{
"signal": signal,
"action_started": action_started,
"action_finished": action_finished,
"restore_started": restore_started,
"restore_finished": restore_finished,
"timings": timings,
}
)
if signal == "api":
candidates = [
r
for r in load.requests
if action_finished <= r[0] <= restore_started
and r[2] >= 500
]
assert candidates
items = wait(
lambda identity=candidates[0][4]: spans(identity)
)
assert any(
s["name"] == "db.postgresql"
and s["duration_seconds"] >= 2
and s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
for s in items
)
report["database_diagnostic_trace"] = items
else:
assert (
max(
s["signals"]["oldest_job_age"] or 0 for s in points
)
> 20
)
assert (
max(s["signals"]["queue_depth"] or 0 for s in points)
> 0
)
accepted = [
r
for r in load.uploads
if action_finished <= r[0] <= restore_started
and r[2] == 202
]
assert accepted
report["worker_recovered_trace"] = wait(
lambda identity=accepted[0][4]: spans(identity)
)
report["worker_recovered_attachment"] = accepted[0][3]
outage_before = snapshot("backend_before")
outage_started = time.time()
compose("stop", "--timeout", "1", "loki", "tempo")
outage = hold("backend_unavailable", 90)
points = [
s
for s in report["samples"]
if s["phase"] == "backend_unavailable"
]
assert max(
s["signals"]["collector_failed_spans"] or 0 for s in points
) > (outage_before["signals"]["collector_failed_spans"] or 0)
assert all(
s["signals"]["collector_queue"] is not None
and s["signals"]["collector_queue_capacity"] is not None
and s["signals"]["collector_memory"] is not None
for s in points
)
assert all(
(s["signals"]["collector_queue"] or 0)
<= (s["signals"]["collector_queue_capacity"] or 0)
for s in points
)
assert (
max(s["signals"]["collector_memory"] for s in points)
< report["container_limits"]["otel"]["mem_limit"]
)
assert not BURN_ALERTS.intersection(outage["firing"])
during = [
r for r in load.requests if r[1] == "backend_unavailable"
]
assert during and all(r[2] == 201 for r in during)
before_ready = sql(
"SELECT count(*) FROM attachments WHERE state='ready'"
)
compose("up", "-d", "--no-deps", "--no-recreate", "loki", "tempo")
wait(lambda: fetch("loki", "/ready", json_result=False))
wait(lambda: fetch("tempo", "/ready", json_result=False))
hold("backend_recovery", 60)
restored = next(
r
for r in reversed(load.uploads)
if r[1] == "backend_recovery" and r[2] == 202
)
wait(
lambda: any(
s["name"] == "attachment.process"
for s in spans(restored[4])
)
)
wait(lambda: logs(restored[4]).get("data", {}).get("result"))
report["backend_outage"] = {
"started": outage_started,
"finished": outage["end"],
"before": outage_before,
"after": snapshot("backend_after"),
"ready_count_before_backend_recovery": int(before_ready),
"successful_writes_during_outage": len(during),
"restored_trace_id": restored[4],
"queue_bounded": True,
"collector_rss_below_limit": True,
"send_failure_counter_counts_attempts_not_unique_loss": True,
}
remaining = args.minimum_seconds - (
time.monotonic() - study_started
)
if remaining > 0:
hold("steady_after_recovery", remaining)
load.close()
load = None
# Read through the public API: a successful SQL insert alone is not
# proof that every acknowledged message remains accessible.
seen, cursor = set(), None
while True:
page = client.get(
f"/api/v1/chats/{chat}/messages",
headers=headers,
params={
"limit": 100,
**({"cursor": cursor} if cursor else {}),
},
)
page.raise_for_status()
data = page.json()
ids = [row["id"] for row in data["items"]]
assert len(ids) == len(set(ids)) and not seen.intersection(ids)
seen.update(ids)
cursor = data.get("next_cursor")
if not cursor:
break
expected_messages = {
r[5] for r in workload.requests if r[2] == 201
}
assert seen == expected_messages
accepted_uploads = [r for r in workload.uploads if r[2] == 202]
for row in accepted_uploads:
def completed(identity=row[3]):
response = client.get(
f"/api/v1/attachments/{identity}", headers=headers
)
return (
response.status_code == 200
and response.json()["status"] == "ready"
)
wait(completed, timeout=120)
for row in accepted_uploads[
:: max(1, len(accepted_uploads) // 10)
]:
response = client.get(
f"/api/v1/attachments/{row[3]}/content", headers=headers
)
assert (
response.status_code == 200
and response.content == workload.raw
)
report["preservation"] = {
"acknowledged_messages": len(expected_messages),
"accessible_messages": len(seen),
"accepted_uploads": len(accepted_uploads),
"all_accepted_uploads_ready": True,
"sampled_originals_match": True,
}
report["rules"] = fetch("prometheus", "/api/v1/rules")
report["final_30m_sli"] = queries(
{
"api_good_ratio": f"1 - sum(increase({ERRORS}[30m])) / clamp_min(sum(increase({RESPONSES}[30m])),1)",
"ready_good_ratio": 'sum(increase(messenger_slo_decisions_total{outcome="good"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)',
}
)
report["complete"] = True
finally:
try:
db_delay(False)
finally:
compose(
"up",
"-d",
"--no-deps",
"--no-recreate",
"worker",
"loki",
"tempo",
)
if load is not None:
load.close()
report["elapsed_seconds"] = round(
time.monotonic() - study_started, 3
)
request_path = output.with_name(output.stem + "-requests.csv")
with request_path.open("w") as stream:
writer = csv.writer(stream)
writer.writerow(
[
"timestamp",
"phase",
"status",
"seconds",
"trace_id",
"message_id",
]
)
writer.writerows(workload.requests)
report["requests_csv"] = request_path.name
report["request_statuses"] = dict(
Counter(str(r[2]) for r in workload.requests)
)
report["uploads"] = workload.uploads
save()
print(
"PASS: short/sustained SLO incidents, detection/resolution and controlled false paging"
)
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
docker compose restart api postgres cache s3 broker relay worker lb
docker compose up -d --no-build --no-recreate --wait
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 scripts/cluster.py rolling
+137
View File
@@ -0,0 +1,137 @@
import json
import os
import subprocess
import time
from pathlib import Path
from uuid import uuid4
import httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8086")
def command(args, *, input=None):
result = subprocess.run(
args, cwd=ROOT, input=input, capture_output=True, text=True
)
if result.returncode:
raise RuntimeError(
"Command failed: "
+ args[0]
+ "; "
+ result.stderr.replace(str(ROOT), "<repository>")
)
return result.stdout.strip()
def compose(*args):
return command(["docker", "compose", *args])
def sql(query, *, database="messenger"):
return compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
database,
"-A",
"-t",
"-v",
"ON_ERROR_STOP=1",
"-c",
query,
)
def cache(command_name, *args):
script = (
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
'exec valkey-cli "$@"'
)
return compose(
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
)
def ready(base=BASE, *, timeout=30):
end = time.monotonic() + timeout
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
while time.monotonic() < end:
try:
if client.get("/health/ready").status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.2)
raise TimeoutError("API readiness")
def actor(client):
username = "verify_" + uuid4().hex[:20]
password = "Synthetic-" + uuid4().hex + ":🌍"
result = client.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Synthetic",
"password": password,
},
)
assert result.status_code == 201
user = result.json()
session = client.post(
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
)
assert session.status_code == 201
return user, password, session.json()
def auth(session):
return {"Authorization": "Bearer " + session["token"]}
def extra_api(*, ttl=3600, pool_size=1):
name = "mtusi-lab6-test-" + uuid4().hex[:10]
compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"-e",
"SESSION_TTL_SECONDS=" + str(ttl),
"-e",
"DB_POOL_SIZE=" + str(pool_size),
"-p",
"127.0.0.1::8080",
"api",
)
try:
info = json.loads(command(["docker", "inspect", name]))[0]
base = (
"http://127.0.0.1:"
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
)
ready(base)
return name, base
except Exception:
command(["docker", "rm", "-f", name])
raise
def instances():
data = {}
for role in ["api", "postgres", "cache"]:
cid = compose("ps", "-q", role)
info = json.loads(command(["docker", "inspect", cid]))[0]
data[role] = {
"container": cid[:12],
"pid": info["State"]["Pid"],
"started_at": info["State"]["StartedAt"],
}
return data
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 scripts/cluster.py stop
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
docker compose stop --timeout 1 worker
+339
View File
@@ -0,0 +1,339 @@
import argparse
import atexit
import json
import os
import socket
import subprocess
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
PROGRESS = {}
def save_incomplete():
receipt = PROGRESS.get("receipt")
if receipt and receipt.get("result") != "PASS":
target = PROGRESS["output"].with_name(
PROGRESS["output"].stem + "-failed.json"
)
if target.exists():
target = target.with_name(
target.stem + "-" + uuid4().hex[:8] + target.suffix
)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(
json.dumps(
{**receipt, "result": "INCOMPLETE"},
ensure_ascii=False,
indent=2,
)
+ "\n"
)
atexit.register(save_incomplete)
def redact(text, cwd):
directory = cwd / "secrets"
if directory.exists():
for path in directory.iterdir():
if path.is_file():
value = path.read_text().strip()
if len(value) >= 12:
text = text.replace(value, "<secret>")
return (
text.replace(str(cwd), "<clean-clone>")
.replace(str(ROOT), "<repository>")
.replace(str(Path.home()), "<home>")
)
def run(args, *, cwd, env, expected=0):
print("CHECK: " + redact(" ".join(args[:3]), cwd), flush=True)
result = subprocess.run(
args, cwd=cwd, env=env, capture_output=True, text=True
)
if result.returncode != expected:
if "receipt" in PROGRESS:
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args), cwd)
raise RuntimeError(redact(result.stdout + result.stderr, cwd))
return redact(result.stdout.strip(), cwd)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/clean-clone.json")
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise SystemExit("Choose new output path")
assert not subprocess.check_output(
["git", "status", "--porcelain"], cwd=ROOT
)
sha = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
).strip()
identity = uuid4().hex[:10]
project = "mtusi-lab6-clean-" + identity
with socket.socket() as reservation:
reservation.bind(("127.0.0.1", 0))
public_port = str(reservation.getsockname()[1])
with socket.socket() as grafana_reservation:
grafana_reservation.bind(("127.0.0.1", 0))
grafana_port = str(grafana_reservation.getsockname()[1])
env = {
**os.environ,
"COMPOSE_PROJECT_NAME": project,
"API_IMAGE": "mtusi-messenger:lab6-clean-" + identity,
"PUBLIC_PORT": public_port,
"GRAFANA_PORT": grafana_port,
"VCS_REF": sha,
"COMPOSE_FILE": "compose.yaml:compose.test.yaml",
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
receipt = {
"date": datetime.now(UTC).isoformat(),
"source_commit": sha,
"fresh_secrets_environment_and_volumes": True,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": int(public_port),
}
PROGRESS.update(receipt=receipt, output=output)
with tempfile.TemporaryDirectory(prefix="mtusi-lab6-clean-") as temporary:
clone = Path(temporary) / "lab6"
run(
[
"git",
"clone",
"--no-hardlinks",
"--branch",
"solution/bvt2301-maslov",
str(ROOT),
str(clone),
],
cwd=ROOT,
env=env,
)
for name in [".env", ".venv", "secrets"]:
assert not (clone / name).exists()
run(["uv", "sync", "--locked"], cwd=clone, env=env)
receipt["static_checks"] = run(
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
cwd=clone,
env=env,
)
receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env)
run(["make", "bootstrap"], cwd=clone, env=env)
try:
run(
["docker", "compose", "up", "-d", "--build", "--wait"],
cwd=clone,
env=env,
)
env["BASE_URL"] = "http://127.0.0.1:" + public_port
receipt["smoke"] = run(
["make", "test", "BASE_URL=" + env["BASE_URL"]],
cwd=clone,
env=env,
)
run(
[
".venv/bin/python",
"-m",
"scripts.verify_cluster",
"--output",
"evidence/clean-cluster.json",
],
cwd=clone,
env=env,
)
for command_args in [
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"seed",
"--seed",
"evidence/clean-gc-seed.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_pipeline",
"--output",
"evidence/clean-pipeline.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"verify",
"--seed",
"evidence/clean-gc-seed.json",
"--output",
"evidence/clean-cleanup.json",
],
]:
run(command_args, cwd=clone, env=env)
receipt["pipeline"] = json.loads(
(clone / "evidence/clean-pipeline.json").read_text()
)
receipt["cleanup"] = json.loads(
(clone / "evidence/clean-cleanup.json").read_text()
)
receipt["cluster"] = json.loads(
(clone / "evidence/clean-cluster.json").read_text()
)
receipt["http_tests"] = run(
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
)
for module, extra in [
("verify_telemetry", []),
(
"verify_observability_persistence",
["--causal-evidence", "evidence/clean-telemetry.json"],
),
]:
name = (
"telemetry"
if module == "verify_telemetry"
else "observability-persistence"
)
run(
[
".venv/bin/python",
"-m",
"scripts." + module,
"--output",
"evidence/clean-" + name + ".json",
*extra,
],
cwd=clone,
env=env,
)
receipt[name] = json.loads(
(clone / ("evidence/clean-" + name + ".json")).read_text()
)
import hashlib
expected = {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (clone / "src").glob("*.py")
}
ids = run(
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
).split()
assert len(ids) == 2
code = (
"import hashlib,json,pathlib; print(json.dumps({p.name:"
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
)
ids += run(
[
"docker",
"compose",
"ps",
"-q",
"worker",
"relay",
"reconciler",
"controller",
"observer",
],
cwd=clone,
env=env,
).split()
assert len(ids) == 7
for cid in ids:
actual = json.loads(
run(
["docker", "exec", cid, "python", "-c", code],
cwd=clone,
env=env,
)
)
assert actual == expected
info = json.loads(
run(["docker", "inspect", cid], cwd=clone, env=env)
)[0]
assert (
info["Config"]["Labels"][
"org.opencontainers.image.revision"
]
== sha
)
assert not info["HostConfig"]["PortBindings"]
receipt[
"all_seven_application_roles_runtime_hashes_and_revision_match"
] = True
receipt["fsck"] = run(
["git", "fsck", "--full"], cwd=clone, env=env
)
receipt["checks_completed"] = True
except BaseException:
receipt["partial_evidence"] = {}
for name in [
"telemetry",
"observability-persistence",
"cluster",
"pipeline",
"cleanup",
"gc-seed",
]:
partial = clone / "evidence" / ("clean-" + name + ".json")
if partial.exists():
receipt["partial_evidence"][name] = json.loads(
redact(partial.read_text(), clone)
)
receipt["failure_diagnostics"] = {}
for label, arguments in [
("services", ["ps", "-a", "--format", "json"]),
(
"logs",
[
"logs",
"--no-color",
"--tail",
"120",
"lb",
"api",
"s3",
"migrate",
"cache",
],
),
]:
diagnostic = subprocess.run(
["docker", "compose", *arguments],
cwd=clone,
env=env,
capture_output=True,
text=True,
)
receipt["failure_diagnostics"][label] = redact(
diagnostic.stdout + diagnostic.stderr, clone
)
raise
finally:
assert project.startswith("mtusi-lab6-clean-")
run(
["docker", "compose", "down", "--volumes", "--remove-orphans"],
cwd=clone,
env=env,
)
receipt["result"] = "PASS"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
)
if __name__ == "__main__":
main()
+223
View File
@@ -0,0 +1,223 @@
"""Real S3 retention test: seed is separate from verification output."""
import argparse
import json
import time
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
from scripts.verify_pipeline import inside, operator, wait
def seed(path):
if path.exists():
raise ValueError("Choose a new seed file")
ready()
compose("stop", "--timeout", "1", "reconciler")
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c:
user, _, session = actor(c)
headers = auth(session)
result = c.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Cleanup synthetic seed",
"member_ids": [user["id"]],
},
)
assert result.status_code == 201
chat = result.json()["id"]
r = c.post(
f"/api/v1/chats/{chat}/attachments",
headers={**headers, "Content-Type": "image/png"},
params={"filename": "retained.png"},
content=raw,
)
assert r.status_code == 202
attachment = r.json()["id"]
wait(
lambda: (
c.get(
"/api/v1/attachments/" + attachment, headers=headers
).json()["status"]
== "ready"
)
)
job = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
data = {
"seed_id": uuid4().hex,
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"ready_attachment": attachment,
"ready_job": job,
"chat": chat,
"owner": user["id"],
}
# Real old v1 objects will become obsolete after v2 is ready.
second = operator(job, version=2)
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))"
)
)
data["version2_job"] = second
data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan"
data["failed_id"] = str(uuid4())
data["active_id"] = str(uuid4())
data["reservation_id"] = str(uuid4())
data["expired_reservation_id"] = str(uuid4())
data["failed_source"] = f"objects/staging/{data['failed_id']}/original"
data["active_source"] = f"objects/staging/{data['active_id']}/original"
data["reservation_key"] = (
f"objects/staging/{data['reservation_id']}/original"
)
data["expired_reservation_key"] = (
f"objects/staging/{data['expired_reservation_id']}/original"
)
inside(f"""from pathlib import Path
raw=Path('/app/src/__init__.py').read_bytes() or b'fixture'
keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])}
for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest())
with db.connection() as c, c.transaction():
for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]:
key='objects/staging/'+identity+'/original'
c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None))
c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None))
c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r}))
print('true')""")
data["seeded_unix"] = time.time()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: retention seed created; real S3 age must reach300s", flush=True
)
def verify(seed_path, output):
if output.exists():
raise ValueError("Choose a new output path")
data = json.loads(seed_path.read_text())
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"seed": seed_path.name,
"complete": False,
}
try:
compose("stop", "--timeout", "1", "reconciler")
def age():
return inside(
f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))"
)
while age() < 302:
time.sleep(min(5, 303 - age()))
report["real_s3_age_seconds"] = age()
fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan"
inside(
f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')"
)
oldkeys = [
data["orphan"],
data["failed_source"],
data["expired_reservation_key"],
] + [
f"objects/{data['ready_attachment']}/v1/{v}"
for v in ["original", "thumbnail", "crop", "optimized"]
]
protected = [data["active_source"], data["reservation_key"], fresh] + [
f"objects/{data['ready_attachment']}/v2/{v}"
for v in ["original", "thumbnail", "crop", "optimized"]
]
before = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert all(before.values())
dry = inside(
"print(json.dumps(reconcile(db,store,apply=False,grace=300)))"
)
candidates = {a["key"] for a in dry["actions"]}
assert set(oldkeys) <= candidates and not set(protected) & candidates
afterdry = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert afterdry == before
applied = inside(
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
)
repeat = inside(
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
)
assert not repeat["actions"]
after = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert all(not after[k] for k in oldkeys) and all(
after[k] for k in protected
)
failed = inside(
f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))"
)
assert failed["error_code"] == "source_expired"
try:
operator(failed["id"])
except RuntimeError:
replay_refused = True
else:
raise AssertionError("Expired source replay must fail")
# Expired reservation was removed, so a delayed API cannot commit its handoff.
fenced = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))"
)
assert fenced
report.update(
dry_run=dry,
apply=applied,
repeat=repeat,
before=before,
after=after,
dry_run_unchanged=True,
active_current_and_fresh_protected=True,
expired_source_replay_refused=replay_refused,
expired_writer_fenced=fenced,
complete=True,
)
finally:
# Remove only this fixture's artificial active reservation/job; never real jobs.
inside(
f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')"
)
compose("up", "-d", "--no-deps", "--no-recreate", "reconciler")
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: real retention, dry-run/apply/repeat, current/active/fresh protection",
flush=True,
)
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["seed", "verify"])
p.add_argument("--seed", required=True, type=Path)
p.add_argument("--output", type=Path)
a = p.parse_args()
if a.action == "seed":
seed(a.seed)
else:
if a.output is None:
p.error("output required")
verify(a.seed, a.output)
if __name__ == "__main__":
main()
+347
View File
@@ -0,0 +1,347 @@
"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
import argparse
import json
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts import cluster
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
def snapshot():
ids = compose("ps", "-q").split()
result = []
for c in json.loads(command(["docker", "inspect", *ids])):
role = c["Config"]["Labels"]["com.docker.compose.service"]
entry = {
"role": role,
"container": c["Id"][:12],
"pid": c["State"]["Pid"],
"started_at": c["State"]["StartedAt"],
"ports": c["HostConfig"]["PortBindings"],
"cpu_nano": c["HostConfig"]["NanoCpus"],
"memory_limit": c["HostConfig"]["Memory"],
}
if role == "api":
entry["instance"] = cluster.alias(c["Id"])
assert not entry["ports"]
result.append(entry)
return result
def exercise(client, session, chat, action, *, seconds=15):
path = f"/api/v1/chats/{chat['id']}/messages"
observations, confirmed, errors, reads = [], {}, 0, 0
started = time.monotonic()
finished = threading.Event()
action_result = {}
def operate():
time.sleep(0.5)
try:
action()
except BaseException as exc:
action_result["error"] = type(exc).__name__
finally:
action_result["finished"] = round(time.monotonic() - started, 3)
finished.set()
thread = threading.Thread(target=operate)
thread.start()
key = uuid4().hex
try:
while (
time.monotonic() - started < seconds
or not finished.is_set()
or time.monotonic() - started
< action_result.get("finished", 0) + 2
):
h = {**auth(session), "Idempotency-Key": key}
try:
r = client.post(path, headers=h, json={"text": key})
if r.status_code != 201:
errors += 1
continue
m = r.json()
confirmed[m["id"]] = m
observations.append(
{
"at": round(time.monotonic() - started, 3),
"instance": r.headers["x-instance-id"],
}
)
key = uuid4().hex
read = client.get(
path, headers=auth(session), params={"limit": 100}
)
reads += read.status_code == 200
errors += read.status_code != 200
except httpx.HTTPError:
errors += 1
time.sleep(0.04)
finally:
thread.join(timeout=90)
assert not action_result.get("error"), action_result
found, cursor = [], None
while True:
r = client.get(
path,
headers=auth(session),
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
)
assert r.status_code == 200
page = r.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
by_id = {m["id"]: m for m in found}
assert all(by_id.get(k) == v for k, v in confirmed.items())
texts = [m["text"] for m in found]
assert len(texts) == len(set(texts))
recovery = next(
(
x["at"]
for x in observations
if x["at"] >= action_result["finished"]
),
None,
)
assert recovery is not None and (seconds > 20 or recovery <= 10)
elapsed = time.monotonic() - started
return {
"confirmed_writes": len(confirmed),
"successful_reads": reads,
"errors": errors,
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
"first_success_after_action_seconds": recovery,
"action": action_result,
"observations": observations,
"confirmed_preserved": True,
"duplicate_texts": False,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/cluster-repeat.json")
args = parser.parse_args()
report = {
"date": datetime.now(timezone.utc).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"complete": False,
}
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
try:
ready()
initial = snapshot()
apis = [c for c in initial if c["role"] == "api"]
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
report["before"] = initial
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
user, _, session = actor(client)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={
"title": "Cluster verification",
"member_ids": [user["id"]],
},
)
assert r.status_code == 201
chat = r.json()
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
def repeat(_):
with httpx.Client(
base_url=BASE, trust_env=False, timeout=8
) as c:
r = c.post(
path,
headers={**auth(session), "Idempotency-Key": key},
json={"text": "Same concurrent request"},
)
assert r.status_code == 201
return r.json(), r.headers["x-instance-id"]
with ThreadPoolExecutor(max_workers=10) as pool:
repeated = list(pool.map(repeat, range(30)))
assert len({m[0]["id"] for m in repeated}) == 1
assert len({m[1] for m in repeated}) == 2
report["parallel_replay"] = {
"requests": 30,
"messages": 1,
"instances": sorted({m[1] for m in repeated}),
}
# Сбой второй записи транзакции не должен оставить сообщение.
fault_key = uuid4().hex
fault_text = "Atomic rollback " + fault_key
sql(
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
+ fault_key
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
)
try:
failed = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert failed.status_code == 500
assert (
sql(
"SELECT count(*) FROM messages WHERE text='"
+ fault_text
+ "'"
)
== "0"
)
finally:
sql(
"DROP TRIGGER student_idempo_fault ON idempotency; "
"DROP FUNCTION student_idempo_fault()"
)
retried = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert retried.status_code == 201
report["atomic_rollback_and_5xx_retry"] = True
sql(
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE key='" + fault_key + "'"
)
expired = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": "New retention window"},
)
assert expired.status_code == 201
assert expired.json()["id"] != retried.json()["id"]
report["expired_key_can_be_reused"] = True
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
report["failovers"] = []
for api in apis:
def stop(target=api["instance"]):
env = {**os.environ, "TARGET_INSTANCE_ID": target}
subprocess.run(
["scripts/stop-one.sh"],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Failover", "member_ids": [user["id"]]},
)
result = exercise(client, session, r.json(), stop)
assert api["instance"] not in [
o["instance"] for o in result["observations"][-5:]
]
assert (
command(
[
"docker",
"inspect",
"-f",
"{{.State.Running}}",
api["container"],
]
)
== "false"
)
result["stopped"] = api
report["failovers"].append(result)
compose("up", "-d", "--no-build", "--wait")
ready()
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Rolling", "member_ids": [user["id"]]},
)
before_roll = snapshot()
report["rolling"] = exercise(
client,
session,
r.json(),
lambda: command(["scripts/rolling-update.sh"]),
seconds=45,
)
after_roll = snapshot()
assert not (
{c["container"] for c in before_roll if c["role"] == "api"}
& {c["container"] for c in after_roll if c["role"] == "api"}
)
report["rolling"]["before"] = before_roll
report["rolling"]["after"] = after_roll
replay = repeat(0)[0]
assert replay == repeated[0][0]
report["replay_after_replacement"] = True
before = snapshot()
env = {
**os.environ,
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
for mode, hook in [
("failover", "scripts/stop-one.sh"),
("persistence", "scripts/restart.sh"),
]:
if mode == "persistence":
compose("up", "-d", "--no-build", "--wait")
ready()
before = snapshot()
run = subprocess.run(
[
".venv/bin/python",
"tests/resilience.py",
"--mode",
mode,
"--action-script",
hook,
"--base-url",
BASE,
],
cwd=ROOT,
env=env,
capture_output=True,
text=True,
)
assert run.returncode == 0, run.stdout + run.stderr
report["public_" + mode] = run.stdout.strip()
after = snapshot()
for role in ["postgres", "cache", "s3", "broker", "lb"]:
old = next(c for c in before if c["role"] == role)
new = next(c for c in after if c["role"] == role)
assert old["started_at"] != new["started_at"]
report["after_persistence"] = after
report["complete"] = True
finally:
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: process failover, rolling, replay and full persistence")
if __name__ == "__main__":
main()
+145
View File
@@ -0,0 +1,145 @@
"""Restart actual stores and compare data accepted before the restart."""
import argparse
import json
import os
import time
from datetime import UTC, datetime
import httpx
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
from scripts.observe import fetch, logs, query, spans, wait
from scripts.stand import ROOT, command, compose
def instances():
result = {}
for role in ["prometheus", "loki", "tempo", "grafana"]:
cid = compose("ps", "-q", role)
data = json.loads(command(["docker", "inspect", cid]))[0]
result[role] = {
"started_at": data["State"]["StartedAt"],
"volumes": sorted(
m["Name"] for m in data["Mounts"] if m["Type"] == "volume"
),
"read_only": data["HostConfig"]["ReadonlyRootfs"],
"cap_drop": data["HostConfig"]["CapDrop"],
"user": data["Config"]["User"],
}
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--causal-evidence", default="evidence/telemetry-first.json"
)
parser.add_argument(
"--output", default="evidence/observability-persistence-first.json"
)
args = parser.parse_args()
output = ROOT / args.output
assert not output.exists()
causal = json.loads((ROOT / args.causal_evidence).read_text())
assert causal["complete"]
identity = causal["causal_uploads"][0]["trace_id"]
timestamp = time.time() - 20
historical = query("sum(messenger_http_requests_total)", at=timestamp)
assert historical and spans(identity) and logs(identity)["data"]["result"]
report = {
"date": datetime.now(UTC).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"trace_id": identity,
"historical_query_at": timestamp,
"historical_before": historical,
"before": instances(),
"complete": False,
}
password = (ROOT / "secrets/grafana_password").read_text().strip()
base = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006")
with httpx.Client(
base_url=base, auth=("admin", password), timeout=10, trust_env=False
) as grafana:
preferences = grafana.get("/api/user/preferences")
preferences.raise_for_status()
previous = preferences.json()
temporary = {
**previous,
"theme": "light" if previous.get("theme") != "light" else "dark",
}
try:
changed = grafana.put("/api/user/preferences", json=temporary)
changed.raise_for_status()
compose("restart", "prometheus", "loki", "tempo", "grafana")
for role, path in [
("prometheus", "/-/ready"),
("loki", "/ready"),
("tempo", "/ready"),
]:
wait(
lambda role=role, path=path: fetch(
role, path, json_result=False
),
timeout=120,
)
def grafana_ready():
try:
response = grafana.get("/api/health")
return (
response.status_code == 200
and response.json()["database"] == "ok"
)
except httpx.HTTPError:
return False
wait(grafana_ready, timeout=120)
wait(lambda: spans(identity), timeout=120)
wait(lambda: logs(identity)["data"]["result"], timeout=120)
report["historical_after"] = query(
"sum(messenger_http_requests_total)", at=timestamp
)
assert report["historical_after"] == historical
assert (
grafana.get("/api/user/preferences").json()["theme"]
== temporary["theme"]
)
report["after"] = instances()
for role, before in report["before"].items():
after = report["after"][role]
assert before["started_at"] != after["started_at"]
assert (
before["volumes"] == after["volumes"] and before["volumes"]
)
assert after["read_only"] and after["user"] not in {
"",
"0",
"0:0",
"root",
}
assert "ALL" in after["cap_drop"]
report.update(
stored_trace_survives=True,
stored_logs_survive=True,
historical_metric_survives=True,
grafana_sqlite_preference_survives=True,
complete=True,
)
finally:
try:
grafana.put(
"/api/user/preferences", json=previous
).raise_for_status()
finally:
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: metrics, traces, logs and Grafana state survive real store restarts"
)
if __name__ == "__main__":
main()
+726
View File
@@ -0,0 +1,726 @@
"""Repeatable fault injection for grade 5, track 2; synthetic data only."""
import argparse
import hashlib
import io
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
os.environ.setdefault("no_proxy", "localhost,127.0.0.1")
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
PREFIX = """import io,json,hashlib,time,urllib.request
from uuid import UUID,uuid4
from psycopg.types.json import Jsonb
from src.pipeline import database,broker,envelope,STREAM,GROUP
from src.object_store import ObjectStore,BUCKET
from src.operator import change
from src.reconcile import reconcile
db,store,queue=database(),ObjectStore(),broker()
"""
def inside(code):
cid = compose("ps", "-q", "controller").split()[0]
return json.loads(
command(
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
)
)
def wait(fn, timeout=90):
end = time.monotonic() + timeout
while time.monotonic() < end:
value = fn()
if value:
return value
time.sleep(0.15)
raise TimeoutError("Fault scenario condition not reached")
def stop(role):
compose("stop", "--timeout", "1", role)
def start(role):
compose("up", "-d", "--no-deps", "--no-recreate", role)
def row(job):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))"
)
def inject(job, phase, action, remaining=1):
return inside(
f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))"
)
def hit(identity):
return inside(
f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))"
)
def operator(job, version=None):
return inside(
f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))"
)
def duplicate(job, count=5):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))"
)
def effect_code(job, *, pending=False):
return f"""with db.connection() as c:
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
pending_entries=[]
if {pending!r}:
for item in queue.xpending_range(STREAM,GROUP,'-','+',1000):
entry=item['message_id']
messages=queue.xrange(STREAM,min=entry,max=entry)
if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry)
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))"""
def inspect_effect(job, *, pending=False):
return inside(effect_code(job, pending=pending))
def observe_fault(job, marker, *, pending=False):
code = f"""from src.pipeline import event
with db.connection() as c: event(c,UUID({job!r}),'crash_observer_ready')
end=time.monotonic()+60
while True:
with db.connection() as c: hit=c.execute('SELECT hit_at FROM pipeline_faults WHERE id=%s',({marker},)).fetchone()['hit_at']
if hit: break
if time.monotonic()>end: raise TimeoutError('Crash hook not hit')
time.sleep(.025)
"""
return inside(code + effect_code(job, pending=pending))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/pipeline-repeat.json")
parser.add_argument("--repeats", type=int, default=2)
parser.add_argument(
"--start-at", choices=["all", "versions"], default="all"
)
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise ValueError("Choose a new output path")
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"track": 2,
"scope": args.start_at,
"complete": False,
"cases": [],
}
output.parent.mkdir(parents=True, exist_ok=True)
def save(name, details):
report["cases"].append({"case": name, **details})
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS:", name, flush=True)
try:
ready()
# Check instrumentation is really enabled and production default is off.
cid = compose("ps", "-q", "api").split()[0]
assert (
command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import os;print(os.getenv('PIPELINE_FAULTS'))",
]
)
== "1"
)
stop("reconciler")
with httpx.Client(
base_url=BASE, trust_env=False, timeout=25
) as client:
user, _, session = actor(client)
h = auth(session)
c = client.post(
"/api/v1/chats",
headers=h,
json={
"title": "Pipeline synthetic faults",
"member_ids": [user["id"]],
},
)
assert c.status_code == 201
chat = c.json()["id"]
raw_io = io.BytesIO()
Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save(
raw_io, format="PNG"
)
raw = raw_io.getvalue()
def upload():
r = client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "synthetic.png"},
content=raw,
)
assert r.status_code == 202, r.text
a = r.json()
assert a["status"] == "queued" and not a["variants"]
j = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
return a, j
def metadata(a):
r = client.get("/api/v1/attachments/" + a["id"], headers=h)
assert r.status_code == 200, r.text
return r.json()
def until(a, state="ready", timeout=90):
return wait(
lambda: (
r if (r := metadata(a))["status"] == state else None
),
timeout,
)
def confirm(a, j):
until(a)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.status_code == 200 and downloaded.content == raw
)
m = metadata(a)
assert (
m["sha256"] == hashlib.sha256(raw).hexdigest()
and m["error_code"] is None
)
assert {v["name"] for v in m["variants"]} == {
"thumbnail",
"crop",
"optimized",
}
for v in m["variants"]:
r = client.get(
f"/api/v1/attachments/{a['id']}/content",
headers=h,
params={"variant": v["name"]},
)
assert (
r.status_code == 200
and hashlib.sha256(r.content).hexdigest()
== v["sha256"]
)
with Image.open(io.BytesIO(r.content)) as image:
assert (
image.size
== {
"thumbnail": (256, 160),
"crop": (128, 128),
"optimized": (320, 200),
}[v["name"]]
)
assert not image.getexif()
effect = inspect_effect(j)
assert (
effect["publications"] == 1
and len(effect["variants"]) == 4
and len(effect["physical_keys"]) == 4
)
return effect
if args.start_at == "all":
# Source-only crash before the DB handoff must never be reported as accepted.
stop("worker")
stop("relay")
marker = inject(chat, "api_after_staging", "wait")
before = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(
lambda: client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "unconfirmed.png"},
content=raw,
)
)
wait(lambda marker=marker: hit(marker))
compose("kill", "-s", "SIGKILL", "api")
try:
response = future.result(15)
assert response.status_code != 202
except httpx.HTTPError:
pass
pool.shutdown()
start("api")
ready()
after = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
assert before == after
save(
"staging_before_handoff_crash",
{
"no_202": True,
"no_attachment_or_job_created": True,
"orphan_retained_for_cleanup": True,
},
)
# API durable commit/outbox survives restart without a broker or worker.
stop("broker")
a, j = upload()
linked = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert linked.status_code == 201
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
compose("restart", "api")
ready()
start("broker")
start("relay")
start("worker")
effect = confirm(a, j)
save(
"commit_before_publish_and_worker_stopped",
{
"attachment": a["id"],
"effect": effect,
"queued_link_allowed": True,
"api_and_broker_restarted": True,
"broker_stopped_during_upload": True,
},
)
# Broker-volume survival with a confirmed, still-unconsumed stream entry.
stop("worker")
a, j = upload()
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))"
)
)
stop("relay")
ids = inside("print(json.dumps(queue.xrange(STREAM)))")
compose("kill", "-s", "SIGKILL", "broker")
start("broker")
wait(lambda: inside("print(json.dumps(queue.ping()))"))
restored = inside("print(json.dumps(queue.xrange(STREAM)))")
assert restored == ids, {"before": ids, "after": restored}
start("relay")
start("worker")
save(
"confirmed_broker_entry_survives_sigkill",
{
"effect": confirm(a, j),
"same_stream_entries": True,
"entry_count": len(ids),
},
)
# Confirm succeeded, PG marker did not: relay recovery may publish a duplicate.
stop("worker")
stop("relay")
a, j = upload()
marker = inject(j, "relay_after_publish", "crash")
start("relay")
wait(lambda marker=marker: hit(marker))
time.sleep(0.5)
unpublished = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))"
)
assert unpublished >= 1
start("relay")
start("worker")
save(
"relay_crash_after_xadd_before_pg_marker",
{
"unpublished_after_crash": unpublished,
"effect": confirm(a, j),
},
)
# Repeated before/after effect and commit/ack windows.
for repeat in range(args.repeats):
for phase in [
"before_effect",
"after_effect",
"after_commit",
]:
stop("worker")
a, j = upload()
marker = inject(j, phase, "wait")
# Start observation before the child timeout begins.
# Polling through a new Python/Docker process at each
# step can miss the actual COMMIT-before-ACK window.
with ThreadPoolExecutor(max_workers=1) as hot:
observer = hot.submit(
observe_fault,
j,
marker,
pending=phase == "after_commit",
)
wait(
lambda j=j: inside(
f"with db.connection() as c: n=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='crash_observer_ready'\",({j!r},)).fetchone()['n']\nprint(json.dumps(n>0))"
)
)
start("worker")
at_crash = observer.result(timeout=60)
report["active_case"] = {
"phase": phase,
"repeat": repeat + 1,
"at_crash": at_crash,
}
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2)
+ "\n"
)
assert at_crash["job"]["state"] == (
"ready"
if phase == "after_commit"
else "processing"
)
assert at_crash["publications"] == (
1 if phase == "after_commit" else 0
)
assert len(at_crash["physical_keys"]) == (
0 if phase == "before_effect" else 4
)
if phase == "after_commit":
assert at_crash["pending_stream_entries"], (
"Result must be committed while notification is still unacknowledged"
)
assert not any(
e["event"] == "child_finished"
for e in at_crash["events"]
)
compose("kill", "-s", "SIGKILL", "worker")
start("worker")
effect = confirm(a, j)
if phase == "after_commit":
wait(
lambda j=j: any(
e["event"] == "duplicate_decision"
and e["detail"]["state"] == "ready"
for e in inspect_effect(j)["events"]
)
)
wait(
lambda j=j: inside(
f"print(json.dumps(not any(json.loads(fields.get('job','{{}}')).get('job_id')=={j!r} for _,fields in queue.xrange(STREAM))))"
)
)
effect = inspect_effect(j)
else:
assert effect["job"]["attempt"] >= 2
report.pop("active_case", None)
save(
f"{phase}_sigkill_repeat_{repeat + 1}",
{"at_crash": at_crash, "effect": effect},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient")
start("worker")
effect = confirm(a, j)
assert effect["job"]["attempt"] == 2
save("transient_retry_with_jitter", {"effect": effect})
stop("worker")
a, j = upload()
marker = inject(j, "before_effect", "hang")
start("worker")
wait(lambda: hit(marker))
limits = inside(
f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))"
)
assert limits["address_space_bytes"] == [805306368, 805306368]
assert limits["memory.max"] == "1610612736"
effect = confirm(a, j)
assert any(
e["event"] == "retry"
and e["detail"]["code"] == "worker_timeout"
for e in effect["events"]
)
save(
"hung_child_killed_and_retried",
{"effect": effect, "observed_limits": limits},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "permanent")
start("worker")
failed = until(a, "failed")
assert failed["error_code"] == "invalid_image"
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
r = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert r.status_code == 409
assert operator(j) == j
effect = confirm(a, j)
assert effect["job"]["cycle"] == 2
try:
operator(j)
except RuntimeError:
pass
else:
raise AssertionError("Replay of ready must be rejected")
save(
"permanent_failed_registry_and_audited_replay",
{
"failed": failed,
"effect": effect,
"ready_replay_rejected": True,
},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient", remaining=5)
start("worker")
failed = until(a, "failed", timeout=150)
assert (
failed["error_code"] == "retry_exhausted"
and row(j)["attempt"] == 5
)
operator(j)
save(
"finite_retry_exhaustion_then_explicit_replay",
{"failed": failed, "effect": confirm(a, j)},
)
# Reverse notification order and add concurrent copies: one publication/key.
stop("worker")
stop("relay")
a, j = upload()
b, k = upload()
duplicate(k, 8)
duplicate(j, 8)
start("worker")
start("relay")
ea = confirm(a, j)
confirm(b, k)
duplicate(j, 10)
duplicate(k, 10)
wait(
lambda: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert (
inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"]
)
save(
"reordered_and_duplicate_notifications",
{
"effects": [inspect_effect(j), inspect_effect(k)],
"added_duplicates": 36,
"no_extra_attempt_after_ready": True,
},
)
# Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS.
for repeat in range(args.repeats):
stop("worker")
a, j = upload()
marker = inject(j, "after_effect", "wait")
# Keep one controller process hot throughout the narrow stale-writer window.
code = f"""from src.pipeline import event
with db.connection() as c: event(c,UUID({j!r}),'version_controller_waiting')
end=time.monotonic()+45
while True:
with db.connection() as c: hit=c.execute('SELECT hit_at FROM pipeline_faults WHERE id=%s',({marker},)).fetchone()['hit_at']
if hit: break
if time.monotonic()>end: raise TimeoutError('v1 hook never hit')
time.sleep(.05)
k=change(db,store,job_id=UUID({j!r}),operator='synthetic-verifier',reason='Held stale writer race',version=2)
end=time.monotonic()+10
while True:
with db.connection() as c: ready=c.execute('SELECT state FROM jobs WHERE id=%s',(k,)).fetchone()['state']=='ready'
if ready: break
if time.monotonic()>end: raise TimeoutError('v2 did not finish inside live v1 budget')
time.sleep(.05)
with db.connection() as c:
hashes=c.execute('SELECT name,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=2 ORDER BY name',({a["id"]!r},)).fetchall()
c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))
end=time.monotonic()+10
while True:
with db.connection() as c: stale=c.execute("SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='stale_completion_rejected'",({j!r},)).fetchone()['n']
if stale: break
if time.monotonic()>end: raise TimeoutError('Old completion was not rejected')
time.sleep(.05)
print(json.dumps({{'job_id':k,'hashes':{{r['name']:r['sha256'] for r in hashes}}}}))
"""
with ThreadPoolExecutor(max_workers=1) as hot:
future = hot.submit(inside, code)
wait(
lambda j=j: inside(
f"with db.connection() as c: n=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='version_controller_waiting'\",({j!r},)).fetchone()['n']\nprint(json.dumps(n>0))"
)
)
start("worker")
race = future.result(timeout=45)
k = race["job_id"]
hashes = race["hashes"]
confirm(a, k)
duplicate(j, 12)
wait(
lambda j=j: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert hashes == {
v["name"]: v["sha256"] for v in confirm(a, k)["variants"]
}
assert inspect_effect(j)["publications"] == 0
save(
f"stale_v1_after_v2_repeat_{repeat + 1}",
{
"v1": inspect_effect(j),
"v2": inspect_effect(k),
"new_hashes_unchanged": True,
},
)
# No arbitrary URL/bytes fields in queue envelopes.
invalid = inside(
"bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')"
)
assert invalid
wait(
lambda: inside(
"with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))"
)
)
save(
"malformed_notification_rejected", {"durable_diagnostic": True}
)
# Private bucket, bounded IAM, and download after uploader process dies.
a, j = upload()
confirm(a, j)
key = f"objects/{a['id']}/v1/original"
security = inside(f"""from minio.error import S3Error
from urllib.error import HTTPError
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
except HTTPError as e: anonymous=e.code
else: raise AssertionError('Anonymous access allowed')
denials={{}}
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
try: fn()
except S3Error as e: denials[name]=e.code
else: raise AssertionError('IAM allowed forbidden action')
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
assert security["anonymous"] == 403 and set(
security["iam"].values()
) == {"AccessDenied"}
selected = None
for cid in compose("ps", "-q", "api").split():
alias = command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
]
)
uploader = inside(
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
)
if alias == uploader:
selected = cid
assert selected
command(["docker", "stop", "--time", "1", selected])
ready()
wait(
lambda: (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 200
)
)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.content == raw
and downloaded.headers["X-Instance-Id"] != uploader
)
start("api")
ready()
save(
"private_iam_and_other_api_download",
{
"security": security,
"attachment": a["id"],
"uploader": uploader,
"downloader": downloaded.headers["X-Instance-Id"],
"sha256": hashlib.sha256(raw).hexdigest(),
},
)
report["worker_metrics"] = inside(
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
)
report["audit"] = inside(
"with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))"
)
report["complete"] = True
finally:
for role in ["api", "broker", "relay", "worker", "reconciler"]:
try:
start(role)
except Exception:
pass
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: versioned pipeline scope=" + args.start_at, flush=True)
if __name__ == "__main__":
main()
+340
View File
@@ -0,0 +1,340 @@
"""Actual API-to-worker causality, log correlation, provisioning and privacy."""
import argparse
import hashlib
import io
import json
import os
from datetime import UTC, datetime
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
from scripts.observe import fetch, logs, query, spans, wait
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
def verify_chain(items, trace_id, *, retry):
assert items and {s["trace_id"] for s in items} == {trace_id}
by_id = {s["span_id"]: s for s in items}
producers = [s for s in items if s["kind"] == "SPAN_KIND_PRODUCER"]
consumers = [s for s in items if s["kind"] == "SPAN_KIND_CONSUMER"]
processing = [s for s in items if s["name"] == "attachment.process"]
assert producers and consumers and processing
assert {s["service"] for s in items} >= {
"messenger-api",
"messenger-relay",
"messenger-worker",
}
for consumer in consumers:
assert (
by_id[consumer["parent_span_id"]]["kind"] == "SPAN_KIND_PRODUCER"
)
for process in processing:
assert by_id[process["parent_span_id"]]["kind"] == "SPAN_KIND_CONSUMER"
assert any(
s["name"] == "s3.put" and s["service"] == "messenger-worker"
for s in items
)
assert any(
s["name"] == "db.postgresql" and s["service"] == "messenger-worker"
for s in items
)
if retry:
assert len(processing) >= 2
assert any(
s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
for s in processing
)
return items
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/telemetry-first.json")
args = parser.parse_args()
output = ROOT / args.output
assert not output.exists(), "Choose a new output path"
report = {
"date": datetime.now(UTC).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"complete": False,
}
output.parent.mkdir(parents=True, exist_ok=True)
try:
ready(timeout=90)
for service, path in [
("prometheus", "/-/ready"),
("loki", "/ready"),
("tempo", "/ready"),
("otel", "/"),
]:
wait(lambda s=service, p=path: fetch(s, p, json_result=False))
report["backends_ready"] = True
buffer = io.BytesIO()
Image.new("RGB", (320, 200), (24, 110, 153)).save(buffer, "PNG")
raw = buffer.getvalue()
sentinel = "PrivacyProbe-" + uuid4().hex
with httpx.Client(
base_url=BASE, timeout=15, trust_env=False
) as client:
user, _, session = actor(client)
headers = auth(session)
chat = client.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Synthetic telemetry",
"member_ids": [user["id"]],
},
).json()["id"]
message = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**headers, "Idempotency-Key": uuid4().hex},
json={"text": sentinel},
)
assert message.status_code == 201
cases = []
for retry in [False, True]:
if retry:
compose("stop", "--timeout", "1", "worker")
trace_id = uuid4().hex
trace_header = (
"00-" + trace_id + "-" + uuid4().hex[:16] + "-01"
)
accepted = client.post(
f"/api/v1/chats/{chat}/attachments",
params={"filename": sentinel + ".png"},
headers={
**headers,
"Content-Type": "image/png",
"traceparent": trace_header,
},
content=raw,
)
assert (
accepted.status_code == 202
and accepted.json()["status"] == "queued"
)
attachment = accepted.json()["id"]
job = sql(
f"SELECT id FROM jobs WHERE attachment_id='{attachment}'::uuid"
)
if retry:
sql(
f"INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES('{job}'::uuid,'before_effect','transient',1)"
)
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
def finished(identity=attachment):
response = client.get(
f"/api/v1/attachments/{identity}", headers=headers
)
return (
response.status_code == 200
and response.json()["status"] == "ready"
)
wait(finished, timeout=90)
original = client.get(
f"/api/v1/attachments/{attachment}/content",
headers=headers,
)
assert original.content == raw
def traced(identity=trace_id, retry_expected=retry):
items = spans(identity)
if len(
[s for s in items if s["name"] == "attachment.process"]
) < (2 if retry_expected else 1):
return None
if any(s["kind"] == "SPAN_KIND_CONSUMER" for s in items):
# Independent export batches may arrive in a different
# order. Require the whole chain before accepting it.
try:
return verify_chain(
items, identity, retry=retry_expected
)
except (AssertionError, KeyError):
return None
items = wait(traced, timeout=90)
def correlated(identity=trace_id):
result = logs(identity)
streams = result.get("data", {}).get("result", [])
services = {
x["stream"].get("service_name") for x in streams
}
return (
result
if {"messenger-api", "messenger-worker"} <= services
else None
)
correlated_logs = wait(correlated, timeout=90)
span_ids = {s["span_id"] for s in items}
streams = correlated_logs["data"]["result"]
assert all(
row["stream"].get("span_id") in span_ids for row in streams
)
assert any(
row["stream"].get("request_id")
== accepted.headers["X-Request-Id"]
for row in streams
)
exported = json.dumps(
{"spans": items, "logs": correlated_logs}
)
assert (
sentinel not in exported
and session["token"] not in exported
)
cases.append(
{
"retry": retry,
"attachment_id": attachment,
"job_id": job,
"trace_id": trace_id,
"request_id": accepted.headers["X-Request-Id"],
"original_sha256": hashlib.sha256(raw).hexdigest(),
"attempt": int(
sql(
f"SELECT attempt FROM jobs WHERE id='{job}'::uuid"
)
),
"spans": items,
"correlated_logs": correlated_logs,
}
)
report["causal_uploads"] = cases
assert cases[1]["attempt"] >= 2
report["privacy_sentinels_absent_from_traces_and_logs"] = True
names = wait(
lambda: fetch("prometheus", "/api/v1/label/__name__/values")[
"data"
]
)
required = {
"messenger_http_requests_total",
"messenger_http_duration_seconds_bucket",
"messenger_job_attempts_total",
"messenger_job_duration_seconds_bucket",
"messenger_time_to_ready_seconds_bucket",
"messenger_queue_depth",
"messenger_oldest_job_age_seconds",
"messenger_db_pool_used",
"messenger_db_pool_waiting",
"messenger_worker_busy",
"messenger_worker_capacity",
"messenger_process_cpu_seconds_total",
"messenger_process_memory_bytes",
"messenger_slo_decisions_total",
}
wait(
lambda: (
required
<= set(
fetch("prometheus", "/api/v1/label/__name__/values")[
"data"
]
)
)
)
names = fetch("prometheus", "/api/v1/label/__name__/values")["data"]
report["metric_names"] = [
n for n in names if n.startswith("messenger_")
]
series = fetch(
"prometheus",
"/api/v1/series",
params={"match[]": '{__name__=~"messenger_.*"}'},
)["data"]
forbidden = {
"user_id",
"chat_id",
"message_id",
"attachment_id",
"job_id",
"request_id",
"trace_id",
"filename",
}
assert all(not forbidden.intersection(s) for s in series)
assert all(sentinel not in json.dumps(s) for s in series)
report["metric_series_count"] = len(series)
report["metric_labels_low_cardinality"] = True
url = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006")
password = (ROOT / "secrets/grafana_password").read_text().strip()
with httpx.Client(
base_url=url, auth=("admin", password), trust_env=False, timeout=15
) as grafana:
dashboard = grafana.get("/api/dashboards/uid/messenger-lab6")
dashboard.raise_for_status()
imported = dashboard.json()["dashboard"]
assert len(imported["panels"]) == 18
sources = grafana.get("/api/datasources").json()
assert {s["uid"] for s in sources} == {
"prometheus",
"loki",
"tempo",
}
checks = {}
for uid in ["prometheus", "loki", "tempo"]:
r = grafana.get(f"/api/datasources/uid/{uid}/health")
r.raise_for_status()
assert r.json()["status"] == "OK", uid
checks[uid] = r.json()["status"]
result = grafana.post(
"/api/ds/query",
json={
"queries": [
{
"refId": "A",
"datasource": {
"uid": "prometheus",
"type": "prometheus",
},
"expr": "sum(messenger_http_requests_total)",
"instant": True,
"format": "table",
"intervalMs": 5000,
"maxDataPoints": 10,
}
],
"from": "now-5m",
"to": "now",
},
)
result.raise_for_status()
assert result.json()["results"]["A"]["frames"]
report["grafana"] = {
"dashboard_uid": imported["uid"],
"panels": len(imported["panels"]),
"datasources_health": checks,
"actual_query_frames": len(
result.json()["results"]["A"]["frames"]
),
}
report["quantiles"] = {
str(q): query(
f"histogram_quantile({q},sum by(le)(rate(messenger_http_duration_seconds_bucket[2m])))"
)
for q in [0.5, 0.95, 0.99]
}
assert all(report["quantiles"].values())
report["complete"] = True
finally:
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: causal API/publish/consumer/child/DB/S3, retry, correlated logs, privacy and Grafana"
)
if __name__ == "__main__":
main()
View File
+44
View File
@@ -0,0 +1,44 @@
import json
import logging
import os
import signal
import sys
def early_stop(signum, _frame):
print(
json.dumps({"event": "stopped_before_startup", "signal": signum}),
flush=True,
)
sys.exit(0)
signal.signal(signal.SIGTERM, early_stop)
signal.signal(signal.SIGINT, early_stop)
import uvicorn # noqa: E402
from src.logging_config import JsonFormatter # noqa: E402
def main() -> None:
host = os.environ.get("APP_HOST", "127.0.0.1")
port = int(os.environ.get("APP_PORT", "8080"))
if not 1 <= port <= 65535:
raise ValueError("APP_PORT must be between 1 and 65535")
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
logging.basicConfig(level=logging.INFO, handlers=[handler], force=True)
uvicorn.run(
"src.api:app",
host=host,
port=port,
workers=1,
access_log=False,
timeout_graceful_shutdown=5,
log_config=None,
)
if __name__ == "__main__":
main()
+412
View File
@@ -0,0 +1,412 @@
import hashlib
import json
import logging
import tempfile
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from datetime import datetime
from functools import partial
from urllib.parse import quote
from uuid import UUID, uuid4
from anyio import Semaphore, fail_after, to_thread
from fastapi import FastAPI, Request
from fastapi.encoders import jsonable_encoder
from pydantic import ValidationError
from starlette.exceptions import HTTPException
from starlette.requests import ClientDisconnect
from starlette.responses import JSONResponse, Response, StreamingResponse
from src import images, telemetry
from src.attachments import Attachments
from src.cache import UserCache
from src.cursor import parse_uuid
from src.database import Database
from src.domain import timestamp
from src.errors import DomainError, invalid
from src.object_store import ObjectStore
from src.password import Passwords
from src.repository import PostgresRepository
from src.schemas import CreateChat, CreateMessage, CreateUser, Input
from src.service import MessengerService
from src.settings import Settings
logger = logging.getLogger("messenger")
INSTANCE_ID = uuid4().hex
def response(data: object, *, status: int = 200) -> JSONResponse:
return JSONResponse(
jsonable_encoder(
data, custom_encoder={datetime: timestamp, UUID: str}
),
status_code=status,
)
async def body(request: Request, model: type[Input]) -> Input:
try:
data = await request.body()
return model.model_validate(
json.loads(
data.decode("utf-8"),
parse_constant=lambda _: (_ for _ in ()).throw(ValueError()),
)
)
except ValueError, ValidationError, UnicodeError, RecursionError:
raise invalid() from None
def pagination(request: Request) -> tuple[int, str | None]:
limits = request.query_params.getlist("limit")
cursors = request.query_params.getlist("cursor")
if len(limits) > 1 or len(cursors) > 1:
raise invalid()
raw = limits[0] if limits else "50"
if not raw.isascii() or not raw.isdecimal():
raise invalid()
raw = raw.lstrip("0") or "0"
if len(raw) > 3:
raise invalid()
limit = int(raw)
if not 1 <= limit <= 100:
raise invalid()
return limit, cursors[0] if cursors else None
def service(request: Request) -> MessengerService:
return request.app.state.service
async def run_io(function, **kwargs):
return await to_thread.run_sync(partial(function, **kwargs))
async def actor(request: Request):
return await run_io(
service(request).identify, value=request.headers.get("authorization")
)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
telemetry.configure("messenger-api", instance=INSTANCE_ID)
settings = Settings.load()
database = Database(url=settings.database_url, size=settings.pool_size)
cache = UserCache(url=settings.cache_url)
store = ObjectStore()
try:
await run_io(database.open)
passwords = await run_io(Passwords)
app.state.database = database
app.state.store = store
app.state.upload_slots = Semaphore(2)
app.state.attachments = Attachments(
database=database, store=store, instance=INSTANCE_ID
)
app.state.service = MessengerService(
repository=PostgresRepository(database=database, cache=cache),
passwords=passwords,
session_ttl=settings.session_ttl,
)
app.state.ready = True
yield
finally:
app.state.ready = False
await run_io(store.close)
await run_io(cache.close)
await run_io(database.close)
await run_io(telemetry.stop)
app = FastAPI(
lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None
)
app.add_middleware(telemetry.TelemetryMiddleware)
@app.middleware("http")
async def request_context(request: Request, call_next):
request_id = str(uuid4())
request.state.request_id = request_id
try:
result = await call_next(request)
except Exception:
logger.error(
json.dumps({"event": "internal_error", "request_id": request_id})
)
result = response(
{
"error": {
"code": "internal_error",
"message": "Internal error",
"request_id": request_id,
}
},
status=500,
)
result.headers["X-Request-Id"] = request_id
result.headers["X-Instance-Id"] = INSTANCE_ID
return result
@app.exception_handler(DomainError)
async def domain_error(request: Request, error: DomainError):
result = response(
{
"error": {
"code": error.code,
"message": error.message,
"request_id": request.state.request_id,
}
},
status=error.status,
)
result.headers.update(error.headers)
return result
@app.exception_handler(HTTPException)
async def http_error(request: Request, error: HTTPException):
code = "not_found" if error.status_code == 404 else "invalid_request"
return response(
{
"error": {
"code": code,
"message": "Resource not found"
if error.status_code == 404
else "Invalid request",
"request_id": request.state.request_id,
}
},
status=error.status_code,
)
@app.get("/health/live")
async def live():
return response({"status": "ok"})
@app.get("/health/ready")
async def ready():
if not getattr(app.state, "ready", False):
raise DomainError(503, "unavailable", "Not ready")
await run_io(app.state.database.ping)
await run_io(app.state.store.ping)
return response({"status": "ok"})
@app.post("/api/v1/users")
async def register(request: Request):
data = await body(request, CreateUser)
return response(
await run_io(service(request).register, **data.model_dump()),
status=201,
)
@app.get("/api/v1/users/me")
async def current(request: Request):
return response(await actor(request))
@app.post("/api/v1/chats")
async def create_chat(request: Request):
user = await actor(request)
data = await body(request, CreateChat)
return response(
await run_io(
service(request).create_chat, actor_id=user.id, **data.model_dump()
),
status=201,
)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
user = await actor(request)
limit, cursor = pagination(request)
return response(
await run_io(
service(request).chats,
actor_id=user.id,
limit=limit,
cursor=cursor,
)
)
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(request: Request, chat_id: str):
user = await actor(request)
return response(
await run_io(
service(request).get_chat,
actor_id=user.id,
chat_id=parse_uuid(chat_id),
)
)
@app.post("/api/v1/chats/{chat_id}/messages")
async def send(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
data = await body(request, CreateMessage)
return response(
await run_io(
service(request).send,
actor_id=user.id,
chat_id=cid,
idempotency_key=request.headers.get("idempotency-key"),
**data.model_dump(),
),
status=201,
)
@app.get("/api/v1/chats/{chat_id}/messages")
async def history(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
chat = await run_io(
service(request).get_chat, actor_id=user.id, chat_id=cid
)
limit, cursor = pagination(request)
return response(
await run_io(
service(request).history,
actor_id=user.id,
chat=chat,
limit=limit,
cursor=cursor,
)
)
@app.post("/api/v1/auth/sessions")
async def login(request: Request):
return response(
await run_io(
service(request).login,
authorization=request.headers.get("authorization"),
),
status=201,
)
@app.delete("/api/v1/auth/sessions/current")
async def logout(request: Request):
await run_io(
service(request).logout,
authorization=request.headers.get("authorization"),
)
return Response(status_code=204)
@app.post("/api/v1/chats/{chat_id}/attachments")
async def upload(request: Request, chat_id: str):
user = await actor(request)
cid = parse_uuid(chat_id)
await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid)
names = request.query_params.getlist("filename")
if len(names) != 1:
raise invalid()
name = images.filename(names[0])
declared = (
request.headers.get("content-type", "")
.partition(";")[0]
.strip()
.lower()
)
if declared not in {"image/png", "image/jpeg"}:
raise DomainError(
415, "unsupported_media_type", "PNG or JPEG required"
)
length = request.headers.get("content-length")
if length and (not length.isdecimal() or len(length) > 12):
raise invalid()
if length and int(length) > images.MAX_BYTES:
raise DomainError(413, "payload_too_large", "Byte limit exceeded")
try:
with fail_after(20):
async with app.state.upload_slots:
with tempfile.SpooledTemporaryFile(
max_size=1024 * 1024, mode="w+b"
) as stream:
size = 0
digest = hashlib.sha256()
async for chunk in request.stream():
size += len(chunk)
if size > images.MAX_BYTES:
raise DomainError(
413, "payload_too_large", "Byte limit exceeded"
)
digest.update(chunk)
stream.write(chunk)
await run_io(
images.verify, stream=stream, declared=declared
)
result = await run_io(
app.state.attachments.upload,
chat_id=cid,
owner_id=user.id,
filename=name,
content_type=declared,
size=size,
sha256=digest.hexdigest(),
stream=stream,
)
return response(result, status=202)
except ClientDisconnect:
raise invalid() from None
except TimeoutError:
raise DomainError(
503, "unavailable", "Upload deadline exceeded"
) from None
@app.get("/api/v1/attachments/{attachment_id}")
async def attachment_metadata(request: Request, attachment_id: str):
user = await actor(request)
return response(
await run_io(
app.state.attachments.metadata,
identity=parse_uuid(attachment_id),
actor_id=user.id,
)
)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def attachment_content(request: Request, attachment_id: str):
user = await actor(request)
variants = request.query_params.getlist("variant")
if len(variants) > 1:
raise invalid()
row, upstream = await run_io(
app.state.attachments.content,
identity=parse_uuid(attachment_id),
actor_id=user.id,
variant=variants[0] if variants else "original",
)
def chunks():
try:
while data := upstream.read(65536):
yield data
finally:
upstream.close()
upstream.release_conn()
return StreamingResponse(
chunks(),
media_type=row["content_type"],
headers={
"Content-Length": str(row["size_bytes"]),
"Content-Disposition": "inline; filename*=UTF-8''"
+ quote(row["filename"], safe=""),
},
)
+144
View File
@@ -0,0 +1,144 @@
from uuid import uuid4
from src.errors import DomainError, not_found
from src.pipeline import enqueue, fault
def dto(row, variants=()):
return {
k: row[k]
for k in [
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"created_at",
"updated_at",
]
} | {
"status": row["state"],
"variants": [
{
k: v[k]
for k in [
"name",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
]
}
for v in variants
if v["name"] != "original"
],
"error_code": row["error_code"] if row["state"] == "failed" else None,
}
class Attachments:
def __init__(self, *, database, store, instance):
self.database, self.store, self.instance = database, store, instance
def upload(
self,
*,
chat_id,
owner_id,
filename,
content_type,
size,
sha256,
stream,
):
identity, job_id = uuid4(), uuid4()
key = f"objects/staging/{identity}/original"
with self.database.connection() as conn:
conn.execute(
"INSERT INTO upload_reservations(id,object_key) VALUES(%s,%s)",
(identity, key),
)
self.store.put(
key=key,
stream=stream,
size=size,
content_type=content_type,
sha256=sha256,
)
fault(self.database, chat_id, "api_after_staging")
with self.database.connection() as conn, conn.transaction():
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp() FOR UPDATE",
(identity,),
).fetchone()
if reservation is None:
raise DomainError(
503, "unavailable", "Upload reservation expired"
)
row = conn.execute(
"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,uploader_instance,expires_at) "
"VALUES(%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,clock_timestamp()) RETURNING *",
(
identity,
chat_id,
owner_id,
filename,
content_type,
size,
sha256,
key,
self.instance,
),
).fetchone()
conn.execute(
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,1,%s,'queued')",
(job_id, identity, key),
)
enqueue(conn, job_id)
conn.execute(
"DELETE FROM upload_reservations WHERE id=%s", (identity,)
)
fault(self.database, chat_id, "api_after_commit")
# Captured committed snapshot remains queued even if worker already finished.
return dto(row)
def row(self, *, identity, actor_id):
with self.database.connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id AND m.user_id=%s WHERE a.id=%s",
(actor_id, identity),
).fetchone()
if row is None:
raise not_found()
return row
def metadata(self, *, identity, actor_id):
row = self.row(identity=identity, actor_id=actor_id)
variants = []
if row["state"] == "ready":
with self.database.connection() as conn:
variants = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name",
(identity, row["published_version"]),
).fetchall()
return dto(row, variants)
def content(self, *, identity, actor_id, variant):
row = self.row(identity=identity, actor_id=actor_id)
if row["state"] != "ready":
raise DomainError(409, "conflict", "Attachment is not ready")
with self.database.connection() as conn:
data = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s",
(identity, row["published_version"], variant),
).fetchone()
if data is None:
raise not_found()
if self.store.head(key=data["object_key"]) is None:
raise DomainError(503, "unavailable", "Variant unavailable")
return {**data, "filename": row["filename"]}, self.store.get(
key=data["object_key"]
)
+66
View File
@@ -0,0 +1,66 @@
import json
import threading
import time
from datetime import datetime
from uuid import UUID
import redis
from redis.backoff import NoBackoff
from redis.retry import Retry
from src.domain import User, timestamp
class UserCache:
def __init__(self, *, url: str):
self.client = redis.Redis.from_url(
url,
socket_connect_timeout=0.15,
socket_timeout=0.15,
max_connections=16,
retry=Retry(NoBackoff(), 0),
decode_responses=True,
)
self.disabled_until = 0.0
self.lock = threading.Lock()
def close(self):
self.client.close()
def _command(self, function, *args, **kwargs):
if time.monotonic() < self.disabled_until:
return None
try:
return function(*args, **kwargs)
except redis.RedisError, OSError:
with self.lock:
self.disabled_until = time.monotonic() + 5
return None
def get(self, *, user_id: UUID) -> User | None:
raw = self._command(self.client.get, "user:" + str(user_id))
if raw is None:
return None
try:
row = json.loads(raw)
if UUID(row["id"]) != user_id:
return None
return User(
user_id,
row["username"],
row["display_name"],
datetime.fromisoformat(row["created_at"]),
)
except ValueError, TypeError, KeyError:
return None
def put(self, *, user: User):
raw = json.dumps(
{
"id": str(user.id),
"username": user.username,
"display_name": user.display_name,
"created_at": timestamp(user.created_at),
}
)
self._command(self.client.set, "user:" + str(user.id), raw, ex=30)
+64
View File
@@ -0,0 +1,64 @@
import base64
import binascii
import json
import re
from datetime import datetime
from uuid import UUID
from src.domain import Key, timestamp
from src.errors import invalid
def parse_uuid(value: str) -> UUID:
if not isinstance(value, str) or not re.fullmatch(
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
):
raise invalid()
return UUID(value)
def encode_cursor(*, scope: str, key: Key) -> str:
payload = {
"v": 1,
"scope": scope,
"at": timestamp(key[0]),
"id": str(key[1]),
}
return (
base64.urlsafe_b64encode(
json.dumps(payload, separators=(",", ":")).encode()
)
.decode()
.rstrip("=")
)
def decode_cursor(*, value: str | None, scope: str) -> Key | None:
if value is None:
return None
try:
if (
not value
or len(value) > 2048
or not re.fullmatch(r"[A-Za-z0-9_-]+", value)
):
raise ValueError
data = json.loads(
base64.b64decode(
value + "=" * (-len(value) % 4), altchars=b"-_", validate=True
)
)
if (
not isinstance(data, dict)
or type(data.get("v")) is not int
or data["v"] != 1
or data.get("scope") != scope
or not isinstance(data.get("at"), str)
):
raise ValueError
at = datetime.fromisoformat(data["at"])
if at.tzinfo is None or timestamp(at) != data["at"]:
raise ValueError
return at, parse_uuid(data["id"])
except ValueError, TypeError, KeyError, binascii.Error, RecursionError:
raise invalid() from None
+77
View File
@@ -0,0 +1,77 @@
import time
from contextlib import contextmanager
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from src import telemetry
from src.errors import DomainError
class DeadlineConnection(psycopg.Connection):
"""Клиентский срок действует и при замороженном PostgreSQL."""
def wait(self, gen, interval=0.1, timeout=None):
deadline = getattr(self, "request_deadline", None)
remaining = (
max(0.001, deadline - time.monotonic()) if deadline else 3.0
)
bound = min(remaining, timeout) if timeout is not None else remaining
try:
return super().wait(gen, interval=interval, timeout=bound)
except psycopg.OperationalError:
self.close()
raise
class Database:
def __init__(self, *, url: str, size: int):
self.pool = ConnectionPool(
url,
connection_class=DeadlineConnection,
kwargs={
"autocommit": True,
"row_factory": dict_row,
"connect_timeout": 2,
"options": (
"-c statement_timeout=2500 -c lock_timeout=1000 "
"-c idle_in_transaction_session_timeout=6000"
),
},
min_size=1,
max_size=size,
timeout=1,
open=False,
)
telemetry.register_pool(self.pool)
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
self.pool.close(timeout=2)
@contextmanager
def connection(self):
try:
with (
telemetry.span(
"db.postgresql",
attributes={"db.system.name": "postgresql"},
),
self.pool.connection(timeout=1) as conn,
):
conn.request_deadline = time.monotonic() + 4
try:
yield conn
finally:
conn.request_deadline = None
except psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout:
raise DomainError(
503, "unavailable", "Database unavailable"
) from None
def ping(self):
with self.connection() as conn:
conn.execute("SELECT 1").fetchone()
+45
View File
@@ -0,0 +1,45 @@
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import UUID
def utcnow() -> datetime:
return datetime.now(UTC)
def timestamp(value: datetime) -> str:
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
@dataclass(frozen=True, slots=True)
class User:
id: UUID
username: str
display_name: str
created_at: datetime
@dataclass(frozen=True, slots=True)
class Chat:
id: UUID
title: str
member_ids: tuple[UUID, ...]
created_at: datetime
@dataclass(frozen=True, slots=True)
class Message:
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: tuple[UUID, ...] = ()
type Record = User | Chat | Message
type Key = tuple[datetime, UUID]
def order_key(record: Record) -> Key:
return record.created_at, record.id
+16
View File
@@ -0,0 +1,16 @@
class DomainError(Exception):
def __init__(
self, status: int, code: str, message: str, headers=None
) -> None:
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid() -> DomainError:
return DomainError(400, "invalid_request", "Invalid request")
def not_found() -> DomainError:
return DomainError(404, "not_found", "Resource not found")
+77
View File
@@ -0,0 +1,77 @@
import struct
import threading
import unicodedata
import zlib
from PIL import Image, UnidentifiedImageError
from src.errors import DomainError, invalid
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
DECODE_SLOT = threading.BoundedSemaphore(1)
def filename(value):
if (
not value
or not 1 <= len(value) <= 255
or not value.strip()
or any(
c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"}
for c in value
)
):
raise invalid()
return value
def verify(stream, *, declared):
if declared not in {"image/png", "image/jpeg"}:
raise DomainError(
415, "unsupported_media_type", "PNG or JPEG required"
)
with DECODE_SLOT:
try:
stream.seek(0)
with Image.open(stream) as image:
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(
image.format
)
if actual != declared:
raise DomainError(
415, "unsupported_media_type", "Image type mismatch"
)
if image.width * image.height > MAX_PIXELS:
raise DomainError(
413, "payload_too_large", "Pixel limit exceeded"
)
image.verify()
# JPEG verify alone does not decode entropy data.
stream.seek(0)
with Image.open(stream) as image:
image.load()
except Image.DecompressionBombError:
raise DomainError(
413, "payload_too_large", "Pixel limit exceeded"
) from None
except (
UnidentifiedImageError,
OSError,
ValueError,
SyntaxError,
struct.error,
zlib.error,
EOFError,
):
raise DomainError(
415, "unsupported_media_type", "Invalid image"
) from None
except MemoryError:
raise DomainError(
503, "unavailable", "Image memory budget exceeded"
) from None
finally:
stream.seek(0)
return declared
+17
View File
@@ -0,0 +1,17 @@
import json
import logging
from datetime import UTC, datetime
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
message = record.getMessage()
try:
event = json.loads(message)
if not isinstance(event, dict):
event = {"event": message}
except ValueError:
event = {"event": message}
event["timestamp"] = datetime.now(UTC).isoformat()
event["level"] = record.levelname.lower()
return json.dumps(event, ensure_ascii=False)
+51
View File
@@ -0,0 +1,51 @@
import hashlib
import json
from pathlib import Path
import psycopg
from src.settings import secret
def main():
with psycopg.connect(
secret("DATABASE_URL"), autocommit=True, connect_timeout=3
) as conn:
conn.execute("SELECT pg_advisory_lock(2301002)")
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(name text PRIMARY KEY, checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT now())"
)
for path in sorted(Path("migrations").glob("*.sql")):
content = path.read_bytes()
checksum = hashlib.sha256(content).hexdigest()
row = conn.execute(
"SELECT checksum FROM schema_migrations WHERE name=%s",
(path.name,),
).fetchone()
if row:
if row[0] != checksum:
raise RuntimeError(
"Applied migration checksum mismatch"
)
continue
with conn.transaction():
conn.execute(content.decode())
conn.execute(
"INSERT INTO schema_migrations(name,checksum) "
"VALUES(%s,%s)",
(path.name, checksum),
)
print(
json.dumps(
{"event": "migration_applied", "name": path.name}
)
)
finally:
conn.execute("SELECT pg_advisory_unlock(2301002)")
if __name__ == "__main__":
main()
+86
View File
@@ -0,0 +1,86 @@
from contextlib import contextmanager
import urllib3
from minio import Minio
from minio.error import S3Error
from src import telemetry
from src.errors import DomainError
from src.settings import secret
BUCKET = "messenger"
class ObjectStore:
def __init__(self):
self.http = urllib3.PoolManager(
maxsize=32,
block=True,
timeout=urllib3.Timeout(connect=1, read=3),
retries=urllib3.Retry(total=0),
)
self.client = Minio(
"s3:9000",
access_key=secret("S3_ACCESS_KEY"),
secret_key=secret("S3_SECRET_KEY"),
secure=False,
region="us-east-1",
http_client=self.http,
)
@contextmanager
def operation(self, name="request"):
try:
with telemetry.span("s3." + name, attributes={"operation": name}):
yield
except S3Error, urllib3.exceptions.HTTPError, OSError:
raise DomainError(
503, "unavailable", "Object store unavailable"
) from None
def ping(self):
with self.operation("head_bucket"):
if not self.client.bucket_exists(BUCKET):
raise DomainError(
503, "unavailable", "Object store unavailable"
)
def put(self, *, key, stream, size, content_type, sha256):
stream.seek(0)
with self.operation("put"):
self.client.put_object(
BUCKET,
key,
stream,
length=size,
content_type=content_type,
metadata={"sha256": sha256},
part_size=10 * 1024 * 1024,
num_parallel_uploads=1,
)
def head(self, *, key):
with self.operation("head"):
try:
return self.client.stat_object(BUCKET, key)
except S3Error as exc:
if exc.code in {"NoSuchKey", "NoSuchObject"}:
return None
raise
def get(self, *, key):
with self.operation("get"):
return self.client.get_object(BUCKET, key)
def delete(self, *, key):
with self.operation("delete"):
self.client.remove_object(BUCKET, key)
def objects(self):
with self.operation("list"):
yield from self.client.list_objects(
BUCKET, prefix="objects/", recursive=True
)
def close(self):
self.http.clear()
+96
View File
@@ -0,0 +1,96 @@
"""Bounded DB polling; pending tasks missing the deadline count as bad SLIs."""
import time
from uuid import uuid4
from src import telemetry
from src.pipeline import database
def poll(db, epoch):
with db.connection() as conn, conn.transaction():
# Evaluate unfinished jobs at their deadline, not after completion.
# Exclude decided IDs before LIMIT to avoid starving newer jobs.
conn.execute(
"INSERT INTO job_slo_observations(job_id,outcome) "
"SELECT j.id,CASE WHEN e.committed_at<=j.created_at+interval '10 seconds' "
"THEN 'good' ELSE 'bad' END FROM jobs j "
"LEFT JOIN effect_publications e ON e.job_id=j.id "
"WHERE NOT EXISTS(SELECT 1 FROM job_slo_observations o WHERE o.job_id=j.id) "
"AND j.state<>'obsolete' AND (e.committed_at IS NOT NULL "
"OR j.state='failed' OR j.created_at<=clock_timestamp()-interval '10 seconds') "
"ORDER BY j.created_at LIMIT 128 ON CONFLICT DO NOTHING"
)
queue = conn.execute(
"SELECT count(*) FILTER(WHERE state='queued') depth,"
"COALESCE(max(EXTRACT(epoch FROM clock_timestamp()-created_at)) "
"FILTER(WHERE state IN ('queued','processing')),0) oldest FROM jobs"
).fetchone()
decisions = conn.execute(
"SELECT outcome,count(*) n FROM job_slo_observations GROUP BY outcome"
).fetchall()
completed = conn.execute(
"SELECT event,count(*) n FROM pipeline_events "
"WHERE event IN ('effect_committed','failed') GROUP BY event"
).fetchall()
ready = conn.execute(
"SELECT o.job_id,EXTRACT(epoch FROM e.committed_at-j.created_at) seconds "
"FROM job_slo_observations o JOIN effect_publications e ON e.job_id=o.job_id "
"JOIN jobs j ON j.id=o.job_id WHERE o.histogram_epoch IS DISTINCT FROM %s::uuid "
"ORDER BY e.committed_at LIMIT 128 FOR UPDATE OF o SKIP LOCKED",
(epoch,),
).fetchall()
for row in ready:
conn.execute(
"UPDATE job_slo_observations SET histogram_epoch=%s WHERE job_id=%s",
(epoch, row["job_id"]),
)
values = {
"queue_depth": queue["depth"],
"oldest_job_age": float(queue["oldest"]),
"slo_good": 0,
"slo_bad": 0,
"completed_ready": 0,
"completed_failed": 0,
}
values.update({"slo_" + row["outcome"]: row["n"] for row in decisions})
values.update(
{
"completed_"
+ (
"ready" if row["event"] == "effect_committed" else "failed"
): row["n"]
for row in completed
}
)
telemetry.observe(values)
for row in ready:
telemetry.duration(
"time_to_ready",
float(row["seconds"]),
{"job_type": "attachment.process.v1", "outcome": "ready"},
)
def main():
epoch = str(uuid4())
telemetry.configure("messenger-observer", instance=epoch)
db = database()
try:
while True:
try:
poll(db, epoch)
except Exception:
telemetry.log(
"observer.unavailable",
error=True,
error_code="database_unavailable",
)
time.sleep(2)
finally:
db.close()
telemetry.stop()
if __name__ == "__main__":
main()
+123
View File
@@ -0,0 +1,123 @@
"""Explicit local operator actions. There is deliberately no HTTP replay endpoint."""
import argparse
import json
from uuid import UUID, uuid4
from src.object_store import ObjectStore
from src.pipeline import database, enqueue, event
def change(db, store, *, job_id, operator, reason, version=None):
if (
not operator.strip()
or not reason.strip()
or len(operator) > 100
or len(reason) > 500
):
raise ValueError("Operator and reason are required")
with db.connection() as conn, conn.transaction():
job = conn.execute(
"SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,)
).fetchone()
if not job:
raise ValueError("Unknown job")
attachment = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE",
(job["attachment_id"],),
).fetchone()
if job["pipeline_version"] != attachment["pipeline_version"]:
raise ValueError("Superseded job cannot be replayed")
source = (
job["source_key"]
if attachment["published_version"] is None
else attachment["object_key"]
)
if store.head(key=source) is None:
raise ValueError("Source expired or missing; replay refused")
if version is None:
if job["state"] != "failed":
raise ValueError("Only failed jobs can be replayed")
conn.execute(
"UPDATE jobs SET state='queued',attempt=0,cycle=cycle+1,fence=fence+1,error_code=NULL,next_at=clock_timestamp(),lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s",
(job_id,),
)
action = "replay"
target = job_id
else:
if (
version not in {1, 2}
or version <= attachment["pipeline_version"]
):
raise ValueError("Supported version must increase (1 or 2)")
conn.execute(
"UPDATE jobs SET state='obsolete',fence=fence+1,lease_owner=NULL,lease_until=NULL WHERE id=%s",
(job_id,),
)
target = uuid4()
conn.execute(
"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,%s,%s,'queued')",
(target, attachment["id"], version, source),
)
conn.execute(
"UPDATE attachments SET pipeline_version=%s WHERE id=%s",
(version, attachment["id"]),
)
action = "new_version"
conn.execute(
"UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(attachment["id"],),
)
conn.execute(
"INSERT INTO pipeline_audit(job_id,action,operator,reason) VALUES(%s,%s,%s,%s)",
(target, action, operator, reason),
)
event(conn, target, action, {"previous_job": str(job_id)})
enqueue(conn, target)
return str(target)
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["failed", "replay", "version"])
p.add_argument("--job-id", type=UUID)
p.add_argument("--operator")
p.add_argument("--reason")
p.add_argument("--version", type=int)
a = p.parse_args()
db, store = database(), ObjectStore()
try:
if a.action == "failed":
with db.connection() as conn:
rows = conn.execute(
"SELECT id,attachment_id,pipeline_version,attempt,cycle,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at,id"
).fetchall()
print(json.dumps(rows, default=str))
else:
if not all([a.job_id, a.operator, a.reason]):
p.error("job-id/operator/reason required")
if a.action == "version" and a.version is None:
p.error("version required")
print(
json.dumps(
{
"job_id": change(
db,
store,
job_id=a.job_id,
operator=a.operator,
reason=a.reason,
version=a.version
if a.action == "version"
else None,
)
}
)
)
finally:
db.close()
store.close()
if __name__ == "__main__":
main()
+36
View File
@@ -0,0 +1,36 @@
import threading
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from argon2.low_level import Type
from src.errors import DomainError
class Passwords:
def __init__(self):
self.hasher = PasswordHasher(
time_cost=3, memory_cost=65536, parallelism=4, type=Type.ID
)
self.slots = threading.BoundedSemaphore(2)
self.dummy = self.hasher.hash("Synthetic-dummy-password-not-for-login")
def _run(self, function, *args):
if not self.slots.acquire(timeout=5):
raise DomainError(503, "unavailable", "Authentication busy")
try:
return function(*args)
finally:
self.slots.release()
def hash(self, *, password: str) -> str:
return self._run(self.hasher.hash, password)
def verify(self, *, password: str, encoded: str | None) -> bool:
try:
return (
self._run(self.hasher.verify, encoded or self.dummy, password)
and encoded is not None
)
except VerificationError, InvalidHashError:
return False
+289
View File
@@ -0,0 +1,289 @@
"""Durable job state is authoritative; stream entries are wake-up notifications."""
import json
import os
import random
import re
import time
from datetime import UTC, datetime
from uuid import UUID
import redis
from psycopg.types.json import Jsonb
from src import telemetry
from src.database import Database
from src.settings import integer, secret
STREAM = "attachments"
GROUP = "image-workers"
def database():
db = Database(url=secret("DATABASE_URL"), size=4)
db.open()
return db
def broker():
return redis.Redis.from_url(
secret("BROKER_URL"),
decode_responses=True,
socket_connect_timeout=1,
socket_timeout=3,
max_connections=8,
)
def envelope(row):
return dict(
schema_version=1,
job_id=str(row["id"]),
type="attachment.process.v1",
attachment_id=str(row["attachment_id"]),
pipeline_version=row["pipeline_version"],
attempt=row["attempt"] + 1,
created_at=datetime.now(UTC).isoformat().replace("+00:00", "Z"),
)
def validate_job(data):
required = {
"schema_version",
"job_id",
"type",
"attachment_id",
"pipeline_version",
"attempt",
"created_at",
}
if (
not isinstance(data, dict)
or not required <= data.keys()
or data.keys() - required - {"traceparent", "tracestate"}
):
raise ValueError("Invalid job envelope")
if (
type(data["schema_version"]) is not int
or data["schema_version"] != 1
or data["type"] != "attachment.process.v1"
):
raise ValueError("Invalid job type")
for k in ["job_id", "attachment_id"]:
if not isinstance(data[k], str):
raise ValueError("Invalid job UUID")
UUID(data[k])
for k in ["pipeline_version", "attempt"]:
if type(data[k]) is not int or data[k] < 1:
raise ValueError("Invalid job number")
timestamp = data["created_at"]
if not isinstance(timestamp, str) or not re.fullmatch(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})",
timestamp,
):
raise ValueError("Invalid job timestamp")
datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
if "traceparent" in data:
if not isinstance(data["traceparent"], str) or not re.fullmatch(
r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"]
):
raise ValueError("Invalid trace context")
if "tracestate" in data and (
not isinstance(data["tracestate"], str)
or len(data["tracestate"]) > 512
):
raise ValueError("Invalid trace context")
return data
def event(conn, job_id, name, detail=None):
conn.execute(
"INSERT INTO pipeline_events(job_id,event,detail) VALUES(%s,%s,%s)",
(job_id, name, Jsonb(detail or {})),
)
def enqueue(conn, job_id):
row = conn.execute("SELECT * FROM jobs WHERE id=%s", (job_id,)).fetchone()
context = telemetry.carrier()
if not context and row.get("traceparent"):
context = {"traceparent": row["traceparent"]}
if row.get("tracestate"):
context["tracestate"] = row["tracestate"]
if context:
conn.execute(
"UPDATE jobs SET traceparent=%s,tracestate=%s WHERE id=%s",
(
context.get("traceparent", ""),
context.get("tracestate", ""),
job_id,
),
)
conn.execute(
"INSERT INTO outbox(job_id,payload,due_at) VALUES(%s,%s,%s)",
(job_id, Jsonb(envelope(row) | context), row["next_at"]),
)
def fault(db, target, phase):
if os.getenv("PIPELINE_FAULTS") != "1":
return
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM pipeline_faults WHERE target=%s AND phase=%s AND remaining>0 ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1",
(target, phase),
).fetchone()
if row is None:
return
conn.execute(
"UPDATE pipeline_faults SET remaining=remaining-1,hit_at=clock_timestamp() WHERE id=%s",
(row["id"],),
)
action = row["action"]
if action == "crash":
os._exit(89)
if action == "permanent":
raise PermanentFailure("invalid_image")
if action == "transient":
raise RuntimeError("injected_transient")
if action == "hang":
time.sleep(120)
if action == "wait":
end = time.monotonic() + 60
while time.monotonic() < end:
with db.connection() as conn:
released = conn.execute(
"SELECT released_at FROM pipeline_faults WHERE id=%s",
(row["id"],),
).fetchone()
if released and released["released_at"]:
return
time.sleep(0.05)
raise RuntimeError("fault_wait_timeout")
class PermanentFailure(Exception):
pass
def retry(conn, row, error, *, permanent=False):
maximum = integer("JOB_MAX_ATTEMPTS", 5, 1, 20)
failed = permanent or row["attempt"] >= maximum
delay = random.uniform(0.25, min(30, 2 ** row["attempt"]))
state = "failed" if failed else "queued"
code = error if permanent else ("retry_exhausted" if failed else error)
conn.execute(
"UPDATE jobs SET state=%s,fence=fence+1,lease_owner=NULL,lease_until=NULL,error_code=%s,next_at=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s",
(state, code, delay, row["id"]),
)
conn.execute(
"UPDATE attachments SET state=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s",
(
state,
code if failed else None,
row["attachment_id"],
row["pipeline_version"],
),
)
event(
conn,
row["id"],
"failed" if failed else "retry",
{"attempt": row["attempt"], "code": code, "backoff_seconds": delay},
)
if not failed:
enqueue(conn, row["id"])
def watchdog(db):
with db.connection() as conn, conn.transaction():
rows = conn.execute(
"SELECT * FROM jobs WHERE state='processing' AND lease_until<clock_timestamp() ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 16"
).fetchall()
for row in rows:
retry(conn, row, "lease_expired")
# Recreate a wakeup if its stream entry was lost: never recreate logical jobs.
rows = conn.execute(
"SELECT j.* FROM jobs j WHERE j.state='queued' AND j.next_at<=clock_timestamp() AND NOT EXISTS(SELECT 1 FROM outbox o WHERE o.job_id=j.id AND (o.published_at IS NULL OR o.published_at>clock_timestamp()-interval '15 seconds')) FOR UPDATE OF j SKIP LOCKED LIMIT 16"
).fetchall()
for row in rows:
enqueue(conn, row["id"])
def relay_once(db, queue):
watchdog(db)
with db.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT * FROM outbox WHERE published_at IS NULL AND due_at<=clock_timestamp() ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1"
).fetchone()
if not row:
return False
validate_job(row["payload"])
with telemetry.span(
"attachments publish",
kind=telemetry.SpanKind.PRODUCER,
parent=row["payload"],
attributes={
"job.id": str(row["job_id"]),
"messaging.system": "valkey",
"messaging.destination.name": STREAM,
},
):
# WAITAOF must use the SAME connection as XADD (not two pooled calls).
with queue.client() as dedicated:
entry = dedicated.xadd(
STREAM,
{
"job": json.dumps(
row["payload"] | telemetry.carrier(),
separators=(",", ":"),
)
},
)
confirmation = dedicated.execute_command("WAITAOF", 1, 0, 2000)
if int(confirmation[0]) < 1:
raise RuntimeError(
"Broker durability confirmation timed out"
)
telemetry.log("job.published", job_id=str(row["job_id"]))
fault(db, row["job_id"], "relay_after_publish")
conn.execute(
"UPDATE outbox SET published_at=clock_timestamp() WHERE id=%s",
(row["id"],),
)
event(
conn,
row["job_id"],
"published",
{"entry": entry, "outbox": row["id"]},
)
return True
def relay():
telemetry.configure("messenger-relay")
db, queue = database(), broker()
try:
while True:
try:
if not relay_once(db, queue):
time.sleep(0.2)
except Exception:
print(
json.dumps(
{
"event": "relay_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
time.sleep(1)
finally:
queue.close()
db.close()
telemetry.stop()
if __name__ == "__main__":
relay()
+122
View File
@@ -0,0 +1,122 @@
"""Retention cleanup; default is dry-run. Active intent always protects bytes."""
import argparse
import json
import time
from src.object_store import ObjectStore
from src.pipeline import database
from src.settings import integer
def reconcile(db, store, *, apply=False, grace=None):
grace = (
grace
if grace is not None
else integer("STAGING_RETENTION_SECONDS", 86400, 300, 604800)
)
if grace < 300:
raise ValueError("Retention must be at least 300 seconds")
actions = []
for obj in store.objects():
# Recheck each candidate in a bounded transaction, including late PUT age.
with db.connection() as conn, conn.transaction():
now = conn.execute("SELECT clock_timestamp() now").fetchone()[
"now"
]
if (now - obj.last_modified).total_seconds() < grace:
continue
reservation = conn.execute(
"SELECT * FROM upload_reservations WHERE object_key=%s FOR UPDATE",
(obj.object_name,),
).fetchone()
if reservation and reservation["expires_at"] > now:
continue
jobs = conn.execute(
"SELECT * FROM jobs WHERE source_key=%s OR %s LIKE 'objects/' || attachment_id::text || '/v' || pipeline_version::text || '/%%' FOR UPDATE",
(obj.object_name, obj.object_name),
).fetchall()
if any(j["state"] in {"queued", "processing"} for j in jobs):
continue
current = conn.execute(
"SELECT v.* FROM attachment_variants v JOIN attachments a ON a.id=v.attachment_id AND a.published_version=v.pipeline_version WHERE v.object_key=%s",
(obj.object_name,),
).fetchone()
if current:
continue
if any(
j["state"] == "failed"
and (now - j["updated_at"]).total_seconds() < grace
for j in jobs
):
continue
head = store.head(key=obj.object_name)
if (
head is None
or (now - head.last_modified).total_seconds() < grace
):
continue
action = {
"action": "delete_expired_object",
"key": obj.object_name,
"applied": apply,
}
if apply:
# Taking the reservation lock fences an expired API before deletion.
if reservation:
conn.execute(
"DELETE FROM upload_reservations WHERE id=%s",
(reservation["id"],),
)
store.delete(key=obj.object_name)
for job in jobs:
if (
job["state"] == "failed"
and job["source_key"] == obj.object_name
):
conn.execute(
"UPDATE jobs SET error_code='source_expired' WHERE id=%s",
(job["id"],),
)
conn.execute(
"UPDATE attachments SET error_code='source_expired' WHERE id=%s AND pipeline_version=%s AND state='failed'",
(job["attachment_id"], job["pipeline_version"]),
)
actions.append(action)
return {"apply": apply, "retention_seconds": grace, "actions": actions}
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true")
p.add_argument("--loop", action="store_true")
a = p.parse_args()
db, store = database(), ObjectStore()
try:
while True:
try:
print(
json.dumps(reconcile(db, store, apply=a.apply)), flush=True
)
except Exception:
if not a.loop:
raise
print(
json.dumps(
{
"event": "cleanup_retry",
"code": "dependency_unavailable",
}
),
flush=True,
)
if not a.loop:
break
time.sleep(60)
finally:
db.close()
store.close()
if __name__ == "__main__":
main()
+296
View File
@@ -0,0 +1,296 @@
import hashlib
import json
from uuid import UUID
from psycopg.errors import ForeignKeyViolation, UniqueViolation
from src.cache import UserCache
from src.database import Database
from src.domain import Chat, Key, Message, User
from src.errors import DomainError, invalid, not_found
MESSAGE_COLUMNS = (
"m.*, ARRAY(SELECT attachment_id FROM message_attachments ma "
"WHERE ma.message_id=m.id ORDER BY ma.position) attachment_ids"
)
USER_COLUMNS = "id, username, display_name, created_at"
def user_dto(row) -> User:
return User(
row["id"], row["username"], row["display_name"], row["created_at"]
)
def chat_dto(row) -> Chat:
return Chat(
row["id"], row["title"], tuple(row["member_ids"]), row["created_at"]
)
def message_dto(row) -> Message:
return Message(
row["id"],
row["chat_id"],
row["sender_id"],
row["text"],
row["created_at"],
tuple(row.get("attachment_ids", ())),
)
class PostgresRepository:
def __init__(self, *, database: Database, cache: UserCache):
self.database = database
self.cache = cache
def add_user(self, *, user: User, password_hash: str) -> User:
try:
with self.database.connection() as conn:
conn.execute(
"INSERT INTO users(id,username,display_name,"
"password_hash,created_at) VALUES(%s,%s,%s,%s,%s)",
(
user.id,
user.username,
user.display_name,
password_hash,
user.created_at,
),
)
except UniqueViolation:
raise DomainError(
409, "conflict", "Username already exists"
) from None
return user
def get_user(self, *, user_id: UUID) -> User | None:
cached = self.cache.get(user_id=user_id)
if cached is not None:
return cached
with self.database.connection() as conn:
row = conn.execute(
"SELECT " + USER_COLUMNS + " FROM users WHERE id=%s",
(user_id,),
).fetchone()
if row is None:
return None
user = user_dto(row)
self.cache.put(user=user)
return user
def credentials(self, *, username: str):
with self.database.connection() as conn:
row = conn.execute(
"SELECT "
+ USER_COLUMNS
+ ", password_hash FROM users WHERE username=%s",
(username,),
).fetchone()
if row is None:
return None, None
return user_dto(row), row["password_hash"]
def username_exists(self, *, username: str) -> bool:
with self.database.connection() as conn:
return (
conn.execute(
"SELECT 1 FROM users WHERE username=%s", (username,)
).fetchone()
is not None
)
def add_session(
self, *, digest: bytes, user_id: UUID, created_at, expires_at
):
with self.database.connection() as conn:
conn.execute(
"INSERT INTO sessions(token_digest,user_id,created_at,"
"expires_at) VALUES(%s,%s,%s,%s)",
(digest, user_id, created_at, expires_at),
)
def session_user(self, *, digest: bytes) -> UUID | None:
with self.database.connection() as conn:
row = conn.execute(
"SELECT user_id FROM sessions WHERE token_digest=%s "
"AND expires_at>clock_timestamp()",
(digest,),
).fetchone()
return row["user_id"] if row else None
def delete_session(self, *, digest: bytes):
with self.database.connection() as conn:
conn.execute(
"DELETE FROM sessions WHERE token_digest=%s", (digest,)
)
def add_chat(self, *, chat: Chat, actor_id: UUID) -> Chat:
members = chat.member_ids
if actor_id not in members or len(set(members)) != len(members):
raise invalid()
try:
with self.database.connection() as conn, conn.transaction():
conn.execute(
"INSERT INTO chats(id,title,created_at) VALUES(%s,%s,%s)",
(chat.id, chat.title, chat.created_at),
)
with conn.cursor() as cursor:
cursor.executemany(
"INSERT INTO members(chat_id,user_id,position) "
"VALUES(%s,%s,%s)",
[
(chat.id, user_id, position)
for position, user_id in enumerate(members)
],
)
except ForeignKeyViolation:
raise invalid() from None
return chat
def get_chat(self, *, chat_id: UUID, actor_id: UUID) -> Chat:
with self.database.connection() as conn:
row = conn.execute(
"SELECT c.id,c.title,c.created_at,"
"array_agg(m.user_id ORDER BY m.position) member_ids "
"FROM chats c JOIN members m ON m.chat_id=c.id "
"WHERE c.id=%s AND EXISTS(SELECT 1 FROM members own "
"WHERE own.chat_id=c.id AND own.user_id=%s) GROUP BY c.id",
(chat_id, actor_id),
).fetchone()
if row is None:
raise not_found()
return chat_dto(row)
def add_message(
self, *, message: Message, idempotency_key: str
) -> Message:
# Область включает пользователя, метод и нормализованный путь чата.
scope = f"{message.sender_id}:POST:{message.chat_id}:{idempotency_key}"
lock = int.from_bytes(
hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True
)
fingerprint = hashlib.sha256(
json.dumps(
{
"text": message.text,
"attachment_ids": [str(x) for x in message.attachment_ids],
},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode()
).digest()
try:
with self.database.connection() as conn, conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
prior = conn.execute(
"SELECT i.body_hash,i.message_id,"
"i.expires_at>clock_timestamp() active "
"FROM idempotency i WHERE user_id=%s AND chat_id=%s "
"AND key=%s FOR UPDATE",
(message.sender_id, message.chat_id, idempotency_key),
).fetchone()
if prior and prior["active"]:
if bytes(prior["body_hash"]) != fingerprint:
raise DomainError(409, "conflict", "Key already used")
row = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.id=%s",
(prior["message_id"],),
).fetchone()
return message_dto(row)
if message.attachment_ids:
rows = conn.execute(
"SELECT id,state FROM attachments WHERE chat_id=%s "
"AND id=ANY(%s) FOR SHARE",
(message.chat_id, list(message.attachment_ids)),
).fetchall()
if len(rows) != len(message.attachment_ids):
raise not_found()
if any(row["state"] == "failed" for row in rows):
raise DomainError(
409, "conflict", "Attachment is not ready"
)
conn.execute(
"INSERT INTO messages(id,chat_id,sender_id,"
"text,created_at) "
"VALUES(%s,%s,%s,%s,%s)",
(
message.id,
message.chat_id,
message.sender_id,
message.text,
message.created_at,
),
)
if message.attachment_ids:
with conn.cursor() as cursor:
cursor.executemany(
"INSERT INTO message_attachments("
"message_id,chat_id,"
"attachment_id,position) "
"VALUES(%s,%s,%s,%s)",
[
(message.id, message.chat_id, aid, i)
for i, aid in enumerate(message.attachment_ids)
],
)
conn.execute(
"INSERT INTO idempotency(user_id,chat_id,key,body_hash,"
"message_id,expires_at) VALUES(%s,%s,%s,%s,%s,"
"clock_timestamp()+interval '24 hours') "
"ON CONFLICT(user_id,chat_id,key) DO UPDATE SET "
"body_hash=excluded.body_hash,message_id=excluded.message_id,"
"expires_at=excluded.expires_at",
(
message.sender_id,
message.chat_id,
idempotency_key,
fingerprint,
message.id,
),
)
except ForeignKeyViolation:
raise not_found() from None
return message
def list_chats(
self, *, actor_id: UUID, after: Key | None, limit: int
) -> list[Chat]:
condition = "AND (c.created_at,c.id)>(%s,%s)" if after else ""
parameters = (
[actor_id, *after, limit + 1] if after else [actor_id, limit + 1]
)
with self.database.connection() as conn:
rows = conn.execute(
"SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id "
"FROM members WHERE chat_id=c.id "
"ORDER BY position) member_ids "
"FROM chats c JOIN members own ON own.chat_id=c.id "
"WHERE own.user_id=%s "
+ condition
+ " ORDER BY c.created_at,c.id LIMIT %s",
parameters,
).fetchall()
return [chat_dto(row) for row in rows]
def list_messages(
self, *, chat_id: UUID, actor_id: UUID, after: Key | None, limit: int
) -> list[Message]:
condition = "AND (m.created_at,m.id)>(%s,%s)" if after else ""
parameters = (
[chat_id, *after, limit + 1] if after else [chat_id, limit + 1]
)
with self.database.connection() as conn:
rows = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.chat_id=%s "
+ condition
+ " ORDER BY m.created_at,m.id LIMIT %s",
parameters,
).fetchall()
return [message_dto(row) for row in rows]
+65
View File
@@ -0,0 +1,65 @@
import re
from typing import Annotated
from uuid import UUID
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
StrictStr,
ValidationInfo,
field_validator,
model_validator,
)
from src.cursor import parse_uuid
UuidIn = Annotated[UUID, BeforeValidator(parse_uuid)]
class Input(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator("*", mode="after")
@classmethod
def nonblank(cls, value: object, info: ValidationInfo) -> object:
if isinstance(value, str) and (
(not value.strip() and info.field_name != "text")
or any(0xD800 <= ord(c) <= 0xDFFF for c in value)
):
raise ValueError("Invalid text")
return value
class CreateUser(Input):
username: StrictStr
display_name: StrictStr = Field(min_length=1, max_length=100)
password: StrictStr = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_format(cls, value: str) -> str:
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Invalid username")
return value
class CreateChat(Input):
title: StrictStr = Field(min_length=1, max_length=100)
member_ids: list[UuidIn] = Field(min_length=1, max_length=100, strict=True)
class CreateMessage(Input):
text: StrictStr = Field(max_length=4000)
attachment_ids: list[UuidIn] = Field(
default_factory=list, max_length=10, strict=True
)
@model_validator(mode="after")
def content(self):
if len(set(self.attachment_ids)) != len(self.attachment_ids):
raise ValueError("Duplicate attachment")
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Empty message")
return self
+192
View File
@@ -0,0 +1,192 @@
import base64
import hashlib
import re
import secrets
import threading
from datetime import timedelta
from uuid import UUID, uuid4
from src.cursor import decode_cursor, encode_cursor
from src.domain import Chat, Message, Record, User, order_key, utcnow
from src.errors import DomainError, invalid, not_found
from src.password import Passwords
from src.repository import PostgresRepository
def page(*, rows: list[Record], limit: int, scope: str) -> dict:
items = rows[:limit]
cursor = (
encode_cursor(scope=scope, key=order_key(items[-1]))
if len(rows) > limit
else None
)
return {"items": items, "next_cursor": cursor}
def unauthorized(*, basic=False):
challenge = (
'Basic realm="messenger", charset="UTF-8"'
if basic
else 'Bearer realm="messenger"'
)
return DomainError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": challenge},
)
def bearer(value):
if not value:
raise unauthorized()
scheme, _, token = value.partition(" ")
if scheme.lower() != "bearer" or not re.fullmatch(
r"[A-Za-z0-9_-]{32,256}", token
):
raise unauthorized()
return token
def token_digest(value):
return hashlib.sha256(value.encode()).digest()
class MessengerService:
def __init__(
self,
*,
repository: PostgresRepository,
passwords: Passwords,
session_ttl: int,
) -> None:
self.repository = repository
self.passwords = passwords
self.session_ttl = session_ttl
self.registration_locks = [threading.Lock() for _ in range(64)]
def identify(self, *, value: str | None) -> User:
token = bearer(value)
user_id = self.repository.session_user(digest=token_digest(token))
user = self.repository.get_user(user_id=user_id) if user_id else None
if user is None:
raise unauthorized()
return user
def register(
self, *, username: str, display_name: str, password: str
) -> User:
# Bound lock memory and avoid running Argon2 for every local duplicate.
# Different replicas still race safely through PostgreSQL UNIQUE.
stripe = hashlib.sha256(username.encode()).digest()[0] % 64
lock = self.registration_locks[stripe]
if not lock.acquire(timeout=8):
raise DomainError(503, "unavailable", "Registration busy")
try:
if self.repository.username_exists(username=username):
raise DomainError(409, "conflict", "Username already exists")
encoded = self.passwords.hash(password=password)
return self.repository.add_user(
user=User(uuid4(), username, display_name, utcnow()),
password_hash=encoded,
)
finally:
lock.release()
def login(self, *, authorization: str | None):
try:
scheme, _, encoded = (authorization or "").partition(" ")
if scheme.lower() != "basic":
raise ValueError
plain = base64.b64decode(encoded, validate=True).decode("utf-8")
username, separator, password = plain.partition(":")
if (
not separator
or not re.fullmatch(r"[a-z0-9_]{3,32}", username)
or not 12 <= len(password) <= 128
or not password.strip()
):
raise ValueError
except ValueError, UnicodeError:
raise unauthorized(basic=True) from None
user, password_hash = self.repository.credentials(username=username)
if not self.passwords.verify(password=password, encoded=password_hash):
raise unauthorized(basic=True)
now = utcnow()
expires = now + timedelta(seconds=self.session_ttl)
token = secrets.token_urlsafe(32)
self.repository.add_session(
digest=token_digest(token),
user_id=user.id,
created_at=now,
expires_at=expires,
)
return {
"token": token,
"token_type": "Bearer",
"expires_at": expires,
"user": user,
}
def logout(self, *, authorization: str | None):
self.identify(value=authorization)
self.repository.delete_session(
digest=token_digest(bearer(authorization))
)
def create_chat(
self, *, actor_id: UUID, title: str, member_ids: list[UUID]
) -> Chat:
return self.repository.add_chat(
chat=Chat(uuid4(), title, tuple(member_ids), utcnow()),
actor_id=actor_id,
)
def get_chat(self, *, actor_id: UUID, chat_id: UUID) -> Chat:
return self.repository.get_chat(actor_id=actor_id, chat_id=chat_id)
def send(
self,
*,
actor_id: UUID,
chat_id: UUID,
text: str,
idempotency_key: str | None,
attachment_ids: list[UUID],
) -> Message:
if not idempotency_key or not re.fullmatch(
r"[A-Za-z0-9._:-]{1,128}", idempotency_key
):
raise invalid()
return self.repository.add_message(
message=Message(
uuid4(),
chat_id,
actor_id,
text,
utcnow(),
tuple(attachment_ids),
),
idempotency_key=idempotency_key,
)
def chats(self, *, actor_id: UUID, limit: int, cursor: str | None) -> dict:
scope = f"chats:{actor_id}"
after = decode_cursor(value=cursor, scope=scope)
rows = self.repository.list_chats(
actor_id=actor_id, after=after, limit=limit
)
return page(rows=rows, limit=limit, scope=scope)
def history(
self, *, actor_id: UUID, chat: Chat, limit: int, cursor: str | None
) -> dict:
if actor_id not in chat.member_ids:
raise not_found()
chat_id = chat.id
scope = f"messages:{chat_id}"
after = decode_cursor(value=cursor, scope=scope)
rows = self.repository.list_messages(
actor_id=actor_id, chat_id=chat_id, after=after, limit=limit
)
return page(rows=rows, limit=limit, scope=scope)
+37
View File
@@ -0,0 +1,37 @@
import os
from dataclasses import dataclass, field
from pathlib import Path
def secret(name: str) -> str:
file = os.environ.get(name + "_FILE")
if file:
return Path(file).read_text().strip()
value = os.environ.get(name)
if not value:
raise ValueError("Missing required secret setting: " + name)
return value
def integer(name: str, default: int, low: int, high: int) -> int:
value = int(os.environ.get(name, str(default)))
if not low <= value <= high:
raise ValueError("Invalid setting: " + name)
return value
@dataclass(frozen=True)
class Settings:
database_url: str = field(repr=False)
cache_url: str = field(repr=False)
session_ttl: int = 3600
pool_size: int = 10
@classmethod
def load(cls):
return cls(
secret("DATABASE_URL"),
secret("CACHE_URL"),
integer("SESSION_TTL_SECONDS", 3600, 1, 86400),
integer("DB_POOL_SIZE", 10, 1, 32),
)
+441
View File
@@ -0,0 +1,441 @@
"""Explicit OTLP signals; no HTTP bodies, credentials, SQL or raw URLs."""
import json
import os
import re
import threading
import time
import weakref
from contextlib import contextmanager
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
import psutil
from opentelemetry import trace
from opentelemetry._logs import SeverityNumber
from opentelemetry.exporter.otlp.proto.http._log_exporter import (
OTLPLogExporter,
)
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
OTLPMetricExporter,
)
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter,
)
from opentelemetry.metrics import Observation
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import Histogram, MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import (
ExplicitBucketHistogramAggregation,
View,
)
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import (
TraceContextTextMapPropagator,
)
_state = None
_lock = threading.Lock()
_pools = weakref.WeakSet()
_instruments = {}
_observer = {}
_propagator = TraceContextTextMapPropagator()
BUCKETS = (
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1,
2.5,
5,
10,
20,
60,
120,
)
LOG_FIELDS = {
"request_id",
"job_id",
"attachment_id",
"method",
"route",
"status",
"duration_seconds",
"attempt",
"version",
"fence",
"outcome",
"error_code",
"operation",
"cycle",
}
TRACE_FIELDS = {
"request_id",
"job.id",
"attachment.id",
"pipeline.version",
"job.attempt",
"db.system.name",
"operation",
"messaging.system",
"messaging.destination.name",
"http.route",
"http.request.method",
"http.response.status_code",
"error.type",
}
def enabled():
return os.getenv("OTEL_ENABLED", "0") == "1"
def register_pool(pool):
_pools.add(pool)
def observe(values):
_observer.update(values)
def _pool_stat(key):
stats = [p.get_stats() for p in list(_pools)]
if key == "used":
return sum(
max(0, s.get("pool_size", 0) - s.get("pool_available", 0))
for s in stats
)
return sum(s.get("requests_waiting", 0) for s in stats)
def configure(service, *, instance=None, with_metrics=True):
global _state
if not enabled():
return
with _lock:
if _state is not None:
return
instance = instance or uuid4().hex
resource = Resource.create(
{
"service.name": service,
"service.version": os.getenv("APP_VERSION", "development"),
"service.instance.id": instance,
"deployment.environment.name": "lab",
}
)
endpoint = os.getenv(
"OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel:4318"
).rstrip("/")
options = {"timeout": 0.75}
# Certificate options are also honored by the exporters in the TLS lab.
mp = None
if with_metrics:
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(
endpoint=endpoint + "/v1/metrics", **options
),
export_interval_millis=2000,
export_timeout_millis=1000,
)
mp = MeterProvider(
resource=resource,
metric_readers=[reader],
views=[
View(
instrument_type=Histogram,
aggregation=ExplicitBucketHistogramAggregation(
BUCKETS
),
)
],
)
meter = mp.get_meter("messenger", "1.0")
for name in ["http_requests", "job_attempts"]:
_instruments[name] = meter.create_counter("messenger_" + name)
_instruments["worker_busy"] = meter.create_up_down_counter(
"messenger_worker_busy"
)
for name in ["http_duration", "job_duration", "time_to_ready"]:
_instruments[name] = meter.create_histogram(
"messenger_" + name, unit="s"
)
for key in ["used", "waiting"]:
meter.create_observable_gauge(
"messenger_db_pool_" + key,
callbacks=[lambda _, k=key: [Observation(_pool_stat(k))]],
)
process = psutil.Process()
meter.create_observable_counter(
"messenger_process_cpu",
unit="s",
callbacks=[
lambda _: [Observation(sum(process.cpu_times()[:2]))]
],
)
meter.create_observable_gauge(
"messenger_process_memory",
unit="By",
callbacks=[lambda _: [Observation(process.memory_info().rss)]],
)
def container_memory(_):
p = Path("/sys/fs/cgroup/memory.current")
return [Observation(int(p.read_text()))] if p.exists() else []
meter.create_observable_gauge(
"messenger_container_memory",
unit="By",
callbacks=[container_memory],
)
if service == "messenger-worker":
meter.create_observable_gauge(
"messenger_worker_capacity",
callbacks=[
lambda _: [
Observation(
int(os.getenv("WORKER_CONCURRENCY", "2"))
)
]
],
)
if service == "messenger-observer":
for key in ["queue_depth", "oldest_job_age"]:
meter.create_observable_gauge(
"messenger_" + key,
unit="s" if key.endswith("age") else "",
callbacks=[
lambda _, k=key: [
Observation(
_observer.get(k, 0),
{"queue": "attachments"},
)
]
],
)
for name, prefix in [
("slo_decisions", "slo_"),
("jobs_completed", "completed_"),
]:
meter.create_observable_counter(
"messenger_" + name,
callbacks=[
lambda _, p=prefix: [
Observation(
_observer.get(p + outcome, 0),
{
"outcome": outcome,
"job_type": "attachment.process.v1",
},
)
for outcome in (
["good", "bad"]
if p == "slo_"
else ["ready", "failed"]
)
]
],
)
tp = TracerProvider(
resource=resource,
sampler=ParentBased(
TraceIdRatioBased(float(os.getenv("OTEL_TRACE_RATIO", "1")))
),
)
tp.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", **options),
max_queue_size=512,
max_export_batch_size=64,
schedule_delay_millis=200,
export_timeout_millis=1000,
)
)
lp = LoggerProvider(resource=resource)
lp.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(endpoint=endpoint + "/v1/logs", **options),
max_queue_size=512,
max_export_batch_size=64,
schedule_delay_millis=200,
export_timeout_millis=1000,
)
)
_state = SimpleNamespace(
service=service,
instance=instance,
traces=tp,
metrics=mp,
logs=lp,
tracer=tp.get_tracer("messenger", "1.0"),
logger=lp.get_logger("messenger", "1.0"),
)
def stop():
if _state is None:
return
for provider in [_state.traces, _state.logs, _state.metrics]:
if provider is not None:
provider.force_flush(timeout_millis=1000)
provider.shutdown()
def count(name, value=1, attributes=None):
instrument = _instruments.get(name)
if instrument is not None:
instrument.add(value, attributes or {})
def duration(name, seconds, attributes=None):
instrument = _instruments.get(name)
if instrument is not None:
instrument.record(max(0, seconds), attributes or {})
def carrier():
result = {}
_propagator.inject(result)
if len(result.get("tracestate", "")) > 512:
result.pop("tracestate", None)
return result
@contextmanager
def span(name, *, kind=SpanKind.INTERNAL, parent=None, attributes=None):
tracer = (
_state.tracer if _state is not None else trace.get_tracer("messenger")
)
attrs = {k: v for k, v in (attributes or {}).items() if k in TRACE_FIELDS}
context = _propagator.extract(parent) if parent is not None else None
with tracer.start_as_current_span(
name,
kind=kind,
context=context,
attributes=attrs,
record_exception=False,
set_status_on_exception=False,
) as current:
try:
yield current
except BaseException as exc:
current.set_attribute("error.type", type(exc).__name__)
current.set_status(Status(StatusCode.ERROR, "operation failed"))
raise
def log(event, *, error=False, **fields):
event = (
event
if re.fullmatch(r"[a-z][a-z0-9_.-]{0,79}", event)
else "application.event"
)
attrs = {
k: v[:256] if isinstance(v, str) else v
for k, v in fields.items()
if k in LOG_FIELDS and isinstance(v, (str, int, float, bool))
}
current = trace.get_current_span().get_span_context()
record = {
"event": event,
"timestamp": time.time(),
"severity": "ERROR" if error else "INFO",
"service.name": _state.service if _state else "messenger",
**attrs,
}
if current.is_valid:
record.update(
trace_id=f"{current.trace_id:032x}",
span_id=f"{current.span_id:016x}",
)
print(json.dumps(record, ensure_ascii=False), flush=True)
if _state is not None:
_state.logger.emit(
body=event,
attributes=attrs,
severity_text=record["severity"],
severity_number=SeverityNumber.ERROR
if error
else SeverityNumber.INFO,
)
class TelemetryMiddleware:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
# Health polling does not enter business SLIs or spans.
if scope.get("path", "").startswith("/health/"):
return await self.app(scope, receive, send)
headers = {
k.decode("latin1"): v.decode("latin1")
for k, v in scope.get("headers", [])
if k.lower() in {b"traceparent", b"tracestate"}
}
status = 500
started = time.monotonic()
async def measured_send(message):
nonlocal status
if message["type"] == "http.response.start":
status = message["status"]
await send(message)
with span("HTTP", kind=SpanKind.SERVER, parent=headers) as current:
try:
await self.app(scope, receive, measured_send)
finally:
route = getattr(scope.get("route"), "path", "unmatched")
method = scope.get("method", "UNKNOWN")
if method not in {
"GET",
"POST",
"PUT",
"PATCH",
"DELETE",
"HEAD",
"OPTIONS",
}:
method = "OTHER"
current.update_name(method + " " + route)
current.set_attributes(
{
"http.route": route,
"http.request.method": method,
"http.response.status_code": status,
}
)
if status >= 500:
current.set_status(
Status(StatusCode.ERROR, "HTTP server error")
)
elapsed = time.monotonic() - started
labels = {
"route": route,
"method": method,
"status_class": str(status // 100) + "xx",
}
count("http_requests", attributes=labels)
duration("http_duration", elapsed, labels)
log(
"http.request",
request_id=scope.get("state", {}).get("request_id", ""),
method=method,
route=route,
status=status,
duration_seconds=round(elapsed, 6),
error=status >= 500,
)

Some files were not shown because too many files have changed in this diff Show More