diff --git a/.dockerignore b/.dockerignore
index bc16e55..8eb0c08 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -18,3 +18,5 @@ __pycache__
node_modules
evidence
.DS_Store
+
+build
diff --git a/.env.example b/.env.example
index 2b32655..b3abae7 100644
--- a/.env.example
+++ b/.env.example
@@ -1,30 +1,19 @@
-# Copy to .env for your LOCAL synthetic test environment; never commit .env.
-# These are logical interface suggestions; map actual framework names in REPORT.md.
-APP_HOST=0.0.0.0
-APP_PORT=8080
-APP_ENV=lab
+APP_PORT=18093
+GRAFANA_PORT=13006
+PROMETHEUS_PORT=19096
+LOKI_PORT=13106
+TEMPO_PORT=13206
+COLLECTOR_PORT=18896
SESSION_TTL_SECONDS=3600
-# Choose credentials locally. File paths contain no secret values.
-DATABASE_URL_FILE=/run/secrets/database_url
-# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
-SESSION_STORE=postgres
-CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
-IDEMPOTENCY_TTL_SECONDS=86400
-S3_ENDPOINT=http://s3:9000
-S3_REGION=us-east-1
-S3_BUCKET=messenger-attachments
-S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
-S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
-MAX_UPLOAD_BYTES=10485760
-MAX_IMAGE_PIXELS=40000000
-BROKER_URL_FILE=/run/secrets/broker_url
-WORKER_CONCURRENCY=2
+API_MEMORY=256m
+HTTP_WORKERS=16
+DB_POOL_MAX=8
+DB_ACCESS_MODE=optimized
+WORKER_CONCURRENCY=1
+MAX_PENDING_JOBS=100
JOB_MAX_ATTEMPTS=5
-JOB_TIMEOUT_SECONDS=60
-STAGING_RETENTION_SECONDS=86400
-OTEL_SERVICE_NAME=messenger-api
-# Worker must set a distinct service.name.
-OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
-OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
-OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
+JOB_TIMEOUT_SECONDS=20
+JOB_LEASE_SECONDS=60
+TELEMETRY_ENABLED=1
+TRACE_SAMPLE_RATIO=0.25
diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml
new file mode 100644
index 0000000..11cfb04
--- /dev/null
+++ b/.github/workflows/application.yml
@@ -0,0 +1,24 @@
+name: Application
+on: [push, pull_request]
+permissions:
+ contents: read
+jobs:
+ application:
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ env:
+ BASE_URL: http://127.0.0.1:18093
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ - run: python3 scripts/bootstrap.py
+ - run: docker compose build api
+ - run: docker compose up -d --wait
+ - run: python3 scripts/wait_ready.py
+ - run: make check test test-student BASE_URL=$BASE_URL
+ - run: make test-observability BASE_URL=$BASE_URL
+ - run: make test-persistence BASE_URL=$BASE_URL ACTION_SCRIPT=scripts/restart.sh
+ - if: always()
+ run: docker compose down -v
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..5e287bd
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,13 @@
+FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 MALLOC_ARENA_MAX=2
+WORKDIR /app
+RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY src/ ./src/
+COPY migrations/ ./migrations/
+USER 10001:10001
+EXPOSE 8080
+HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"]
+STOPSIGNAL SIGTERM
+CMD ["python", "src/server.py"]
diff --git a/Makefile b/Makefile
index 5d86d17..82e2a4b 100644
--- a/Makefile
+++ b/Makefile
@@ -28,3 +28,19 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+
+.PHONY: test-student
+test-student:
+ BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p 'test_*.py' -v
+
+.PHONY: test-observability test-incidents test-telemetry-outage
+test-observability:
+ BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py proof
+test-incidents:
+ BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py incidents
+test-telemetry-outage:
+ BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py backend
+
+.PHONY: research
+research:
+ BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability_cost.py
diff --git a/REPORT.md b/REPORT.md
index b97ae42..cf246d0 100644
--- a/REPORT.md
+++ b/REPORT.md
@@ -1,37 +1,350 @@
-# Отчёт — лабораторная 6
+# Лабораторная 6 — наблюдаемость мессенджера
-- Автор, группа: TODO
-- Целевая оценка; выбранный трек 5, если нужен: TODO
-- Commit/PR: TODO после публикации
-- Использованные источники, библиотеки и помощь ИИ: TODO
+**Майорова Софья, БВТ2301. Целевая оценка: 5, трек 1 «Цена наблюдаемости».**
+Дата выполнения: 20 ноября 2026 года. Оценку подтверждает преподаватель.
+
+Продолжена [собственная lab5, PR 7](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/7).
+Перенесены только исходники, миграции, свои тесты и необходимые конфигурации.
+README/COURSE/contracts/tests текущего skeleton lab6 сохранены, assertions публичных проверок не менялись.
+Трек 2 с multi-window burn rate не заявляется.
## Запуск из чистого clone
-TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
+Docker Engine/Compose v2, Linux-контейнеры, Python 3.9+ для host-проверок.
+Сервер и worker Python 3.12.8 внутри Docker. На этом Mac:
-## Архитектура и решения
+```sh
+colima start --profile mtusi-labs
+export DOCKER_CONTEXT=colima-mtusi-labs
+cd "/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab6"
+python3 scripts/bootstrap.py
+docker compose build api
+docker compose up -d --wait
+export BASE_URL=http://127.0.0.1:18093
+python3 scripts/wait_ready.py
+make check test test-student BASE_URL=$BASE_URL
+```
-TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
+На другой машине клонировать публичный fork, выбрать solution/BVT2301-Mayorova и выполнять из корня clone.
+Образ приложения сначала строится через build api: worker, dispatcher, migration и audit используют тот же image.
+Bootstrap генерирует локальные случайные secrets, сохраняет существующие значения и .env.
+Каталог secrets 0700; полный S3 identity config 0600, монтируется только root gateway.
+Пароли, ключи, .env, volumes, backups и build не коммитятся и не попадают в build context.
-## Доказательства критериев
+API: http://127.0.0.1:18093.
+Grafana: http://127.0.0.1:13006/d/messenger-lab6 — автоматический Viewer на localhost.
+GF_USERS_VIEWERS_CAN_EDIT разрешает Explore и временные правки запросов без сохранения dashboard;
+роль остаётся Viewer (настройка deprecated в Grafana 12.1.1, доступна в закреплённой версии).
+Импорт dashboard и трёх datasources выполняет provisioning из Git; ручной настройки не требуется.
+Prometheus http://127.0.0.1:19096, Loki 13106, Tempo 13206, собственные метрики Collector 18896.
+OTLP 4318 не публикуется на host. PostgreSQL/S3/broker/API-реплики тоже не публикуются в обход LB.
+Операторские UI присоединены к edge для корректного loopback port mapping Docker; backend и storage — внутренние сети.
+Наружу в Интернет эти порты не открыты. Grafana не является административной панелью бизнес-данных.
-| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
-| --- | --- | --- | --- |
-| Публичный smoke | make test | TODO | TODO |
-| Остальные пункты уровня 3 | TODO | TODO | TODO |
-| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
-| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
+Остановка с сохранением истории: docker compose down.
+Не удалять volumes в проверке restart/failover. down -v предназначен только для намеренно уничтожаемого отдельного тестового стенда.
+На этой машине старый Docker proxy не разрешался DNS.
+После отдельного разрешения пользователя сохранён backup /etc/docker/daemon.json.lab6-proxy-backup внутри VM,
+удалена неработающая proxy-секция и перезапущен Docker daemon. Реестр проверен по прямому HTTPS.
+Это локальная настройка окружения, не часть приложения и не обязательный шаг для другого clone.
-TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
+## Архитектура
-## Эксперименты и отказы
+```text
+Клиент → HAProxy → API ×2 → PostgreSQL
+ ├→ Redis cache профилей
+ └→ приватный S3 staging
+PostgreSQL jobs/outbox → dispatcher → Redis Streams/AOF → worker → S3 results + DB ready
-TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
+API / dispatcher / worker → OpenTelemetry SDK → OTLP HTTP/protobuf → Collector
+ ├→ metrics → Prometheus → Grafana
+ ├→ logs → Loki → Grafana Explore
+ └→ traces → Tempo → Grafana Explore
+```
-| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
-| --- | --- | --- | --- |
-| TODO | TODO | TODO | TODO |
+Prometheus хранит временные ряды, Loki — структурные логи, Tempo — трассы.
+Grafana выполняет запросы и визуализирует; Collector принимает/обрабатывает/экспортирует и не заменяет долговременные хранилища.
+PostgreSQL остаётся источником истины бизнес-данных и сессий. Наблюдаемость не участвует в COMMIT подтверждённого сообщения/задания.
-## Ограничения и следующие шаги
+Сохранён минимум предыдущих работ: scrypt и сессии с expiry/logout, Unicode/U+0000, ACL чатов,
+keyset, атомарный Idempotency-Key на 24h, две реплики и migration job, Redis profile cache,
+приватные изображения, durable staging/outbox, bounded renderer, at-least-once/retry/lease,
+immutable versioned results, version fencing, failed registry/replay и операторский cleanup.
+API возвращает 202 queued после S3 staging и COMMIT; resize выполняет отдельный worker.
-TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
+Миграции 001–006. Новая 006 добавляет trace_context JSONB к attachment_jobs.
+Миграция выполняется отдельным service под advisory lock; API проверяет наличие 006 до обслуживания.
+Старые jobs получают пустой carrier и продолжают работать; бизнес-API не расширялся.
+
+## Версии и ресурсы
+
+Все Docker images закреплены version + digest в Compose/Dockerfile.
+Python 3.12.8; PostgreSQL 17.2; Redis 7.4.1; SeaweedFS 4.51; HAProxy 3.0.5.
+OTel Python API/SDK/HTTP exporter 1.36.0, semantic conventions 0.57b0;
+Collector contrib 0.132.0; Prometheus 3.5.0; Loki 3.5.3; Tempo 2.8.2; Grafana 12.1.1.
+Все 27 Python distributions, включая транзитивные, закреплены в requirements.txt по успешной сборке.
+
+API ×2: по 1 CPU/256MiB; worker 1 CPU/768MiB, concurrency1; dispatcher 1 CPU/256MiB.
+PostgreSQL 1 CPU/512MiB; broker0.5CPU/192MiB/AOF always; Redis cache0.25CPU/64MiB.
+S3 store1CPU/512MiB и gateway0.5CPU/512MiB; LB0.5CPU/64MiB.
+Collector/Prometheus/Loki/Grafana — по 0.5CPU/256MiB, Tempo0.5CPU/512MiB.
+Сумма CPU-лимитов — overcommit, не выделенные физические ядра.
+Окружение: macOS arm64, Colima Linux VM4vCPU/около4GiB; локальный synthetic workload.
+
+## Схема сигналов и единицы
+
+Ручная SDK-инструментация ограничивает атрибуты разрешённым набором.
+Resource: service.name=messenger-api / messenger-worker / messenger-dispatcher;
+service.version=lab6.1; service.instance.id — случайный alias процесса;
+deployment.environment.name=lab. Дополнительно SDK сообщает собственную версию/язык.
+
+| Реальное имя Prometheus | Значение |
+| --- | --- |
+| http_server_requests_total | Обработанные business requests; method, route template, status_class |
+| http_server_duration_seconds_bucket/count/sum | Полная обработка HTTP, включая отправку ответа |
+| jobs_processed_total | Попытки worker: ready, failed_or_retry, noop, fenced, error; job_type=image |
+| job_processing_duration_seconds_bucket/count/sum | Время попытки, тот же outcome |
+| job_ready_duration_seconds_bucket/count/sum | От created_at durable job перед COMMIT до публикации ready |
+| job_retries_total | Решения retry/terminal failed, outcome=retry или failed |
+| queue_depth | Незавершённые queued/processing в PostgreSQL, включая outbox до broker |
+| queue_oldest_age_seconds | Возраст самой старой такой job; backlog не processing latency |
+| queue_observation_ok | 0 при невозможности наблюдать PostgreSQL backlog |
+| db_pool_used/waiting/capacity | Используемые/ожидающие/максимум соединений API |
+| worker_busy/capacity | Активные попытки/число slots |
+| process_cpu_seconds_total | CPU процесса, cumulative |
+| process_resident_memory_bytes | RSS процесса через /proc/self/statm |
+| container_memory_bytes | Cgroup memory.current, включая renderer и cache |
+| spans_finished/exported/export_failed | Число sampled SDK spans и результат отправки |
+
+В gauge количества unit пустой: иначе unit1 экспортируется Prometheus как _ratio.
+Очередь именуется постоянным attachment-jobs; отдельного динамического queue label не нужно.
+Service instance — resource attribute с ограниченной жизнью процесса; churn реплик всё равно создаёт новые series.
+Health полностью исключён из business metrics/logs/traces. Не распознанные маршруты имеют template unmatched.
+user/chat/message/attachment/request/job/trace IDs, filename, raw URL и текст не являются labels метрик.
+
+Границы duration histogram, секунды:
+0.001,0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2,5,10,20,30,60,+Inf.
+Dashboard агрегирует buckets всех реплик:
+histogram_quantile(0.95, sum by(le)(rate(http_server_duration_seconds_bucket[1m]))).
+p50/p99 построены аналогично; средние p95 реплик не усредняются.
+Точность percentile ограничена границами buckets и количеством наблюдений.
+
+## Трассы, логи и очередь
+
+Handler создаёт SERVER span с method/route/status без headers/body/query.
+Connection context создаёт CLIENT db.transaction span, включающий ожидание пула, SQL и завершение транзакции,
+но без текста SQL, параметров, connection string или сырого текста exception.
+Это время unit of work, а не чистый server-side SQL execution time: S3 PUT под transaction виден дочерним S3 span.
+S3 get span заканчивается при получении SDK response; потоковое чтение Body входит во время родительского HTTP/worker span.
+S3 wrapper записывает только service S3 и имя операции; ключи, filenames, URLs и credentials не записываются.
+Worker consumer span включает processing и реальные дочерние S3/DB spans.
+Внутри изолированного renderer отдельный SDK не запускается.
+
+W3C traceparent/tracestate сохраняются с job в той же транзакции outbox.
+Dispatcher извлекает durable context, создаёт PRODUCER span и вкладывает его carrier в envelope.
+Worker извлекает envelope и создаёт CONSUMER child producer. Retry публикуется с тем же исходным trace,
+новым producer/consumer и номером попытки. Envelope содержит ID/версию/попытку/carrier, а не bytes изображения.
+
+Структурный JSON log содержит timestamp, severity, service.name, event, trace_id, span_id, sampled;
+HTTP — X-Request-Id, template/method/status/duration; job — технический job_id/attempt/outcome.
+Эти технические ID нужны для сквозной диагностики синтетического задания и не индексируются как metric labels.
+LoggingHandler отправляет OTLP logs; Loki принимает native OTLP и хранит остальные поля как structured metadata/body.
+Grafana derived field TraceID ведёт из Loki в Tempo; tracesToLogs фильтрует тот же trace_id.
+Имена полей/шаблоны фиксированы. Тела сообщений, картинки, пароли, Authorization, tokens и signed URLs исключены.
+
+Проверка proof выполняет реальный upload с известным traceparent, ждёт ready,
+проверяет родителя consumer=producer и цепочку предков producer до SERVER upload,
+проверяет worker S3/DB spans и логи всех стадий, связывает response X-Request-Id.
+Отдельно остановлен worker, принят файл, приостановлен реальный S3 gateway, worker запущен:
+наблюдается S3 timeout/retry; после возврата S3 второй consumer достигает ready в той же трассе.
+Canary message/password/token/username/filename не найдены в проверяемом trace и app logs с начала controlled proof (лимит 5000);
+это ограниченная динамическая проверка плюс allowlist кода, не математическое доказательство отсутствия любых утечек.
+Метрики проверены на запрещённые ID labels.
+
+## SLI/SLO и алерты
+
+Полные определения/ограничения — infra/RUNBOOK.md.
+Окно5мин: API availability>=99% обработанных business requests без5xx; health исключён,
+4xx не смешиваются с отказами сервера. Отказ LB/сети до API данным SLI не покрывается.
+Latency SLO: >=95% завершённых ready jobs укладываются в5сек от durable job creation.
+Это условный latency SLO, не доступность pipeline; queued/failed отдельно видны через backlog/outcomes/alerts.
+
+Prometheus правила из infra/alerts.yaml, evaluation2s:
+- APIHighErrorRate: 5xx share за30s>5%, for6s.
+- QueueLag: oldest>10s и depth>0, for6s.
+- QueueObservationUnavailable: observer0, for6s.
+Это короткие учебные operational alerts, не трек2 multi-window burn rate.
+Нет email/Slack или внешней доставки.
+
+## Два инцидента и диагностика
+
+1. Worker остановлен, API принял3 изображения.
+Queue depth/oldest растут, завершения worker прекращаются; upload202 и логи acceptance/publish продолжаются.
+QueueLag firing примерно через22.77сек. После возврата worker все три ready, backlog0 и alert resolved.
+Значения gauges остановленного процесса могут оставаться в scrape cache до30s: это явно учитывается в runbook.
+
+2. Реальная ACCESS EXCLUSIVE блокировка sessions удерживается35s.
+Readiness SELECT1 остаётся зелёной, защищённый endpoint ждёт SQL и получает503 по lock_timeout1.5s.
+APIHighErrorRate firing примерно через13.14сек. В сохранённой трассе db.transaction — долгий/error span;
+это отделяет ожидание PostgreSQL от HTTP handler/resize. После COMMIT блокировки прежний Bearer снова работает,
+30s error window очищается и alert resolved.
+
+Сырые timestamps, firing snapshots, запросы метрик, error latencies и DB trace — evidence/incidents.json.
+Разбор опирается на наблюдения; способ инъекции задокументирован отдельно и не заменяет trace/metric evidence.
+
+## Отказ telemetry backend и конечная буферизация
+
+Tempo остановлен примерно на56сек. За это время API подтвердил135 сообщений и5 файлов,
+worker завершил файлы, все подтверждённые сообщения найдены в истории.
+Collector memory snapshots: примерно58.21–62.05MiB при лимите256MiB; наблюдены156 failed exported spans.
+После восстановления найден новый реальный trace. Потерянная telemetry не пересоздаётся из бизнес-данных.
+
+SDK: BatchSpanProcessor queue512/batch128, logs queue256/batch64, schedule500ms,
+HTTP timeout1s; metric export каждые2s. Отправка асинхронная, очереди ограничены.
+Collector memory_limiter limit192/spike32MiB, batch<=256;
+два exporter queue по128 batches,2consumers, retry budget10s.
+Prometheus exporter имеет expiration30s. Нет безразмерной очереди или бесконечного retry.
+Лимит batches не является точным лимитом байт; SDK/receiver имеют дополнительные ограничения,
+container memory лимит — последняя граница. Локальный тест конечной длительности не доказывает поведение при любом трафике.
+
+Метрики хранятся24h и ограничены256MB TSDB; Loki/Tempo retention24h; Grafana SQLite/provisioning.
+Именованные metrics_data/logs_data/traces_data/grafana_data сохраняются при restart.
+Loki/Tempo retention выполняется асинхронно и не является жёсткой дисковой квотой.
+
+## Исследование цены наблюдаемости
+
+Гипотеза: SDK export/sampling влияет на задержки, CPU/RAM и ingest; уменьшение доли трасс сокращает ingest,
+но часть редких ошибок и медленных запросов не имеет полного trace.
+Методика student-tests/observability_cost.py:
+off, full1.0, head sampling0.10 и0.25; три повтора, порядок режимов циклически сдвигается.
+Одинаковые лимиты,2API,worker concurrency1, fixture320x200.
+Каждый прогон: фиксированные30 scheduled requests/s20s,600 запросов:
+5% upload,15% messages,80% profile reads;16client threads.
+Exporter off оставляет установленные SDK/hooks, но не создаёт providers и не экспортирует signals.
+Это не сравнение с переписанной версией без SDK imports.
+
+Измеряются клиентские p50/p95/p99, подтверждённый RPS/ошибки,
+CPU cgroup API/worker/dispatcher/Collector/хранилищ,
+anonymous/working-set memory samples1Hz,
+точные counter deltas принятых spans/logs/metric points и Collector receive network bytes.
+Последние включают OTLP, Prometheus scrape и backend replies: это не чистый размер protobuf payload.
+CPU/ingest включают ожидание exporter drain5s и проверки ready после основного потока.
+История БД/telemetry сохраняется между режимами; новые API/worker/dispatcher создаются на каждом режиме.
+Пять секунд прогрева/drain, фиксированная среда и смена порядка уменьшают, но не исключают шум/влияние кэша.
+UUID/sampling стохастические; нагрузка локальная и короткая. Это не capacity ceiling и не production SLA.
+
+Первоначальные SQL-lock probes после измерительных окон не синхронизировали момент HTTP с pg_locks:
+часть блокировок успевала завершиться при старте docker exec. Их фактические HTTP статусы сохранены
+в rare_probes основного raw JSON; выдавать их за гарантированные 503 нельзя.
+Для вывода о редких событиях выполнено отдельное исследование student-tests/sampling_retention.py:
+full/10%/25%, три повтора; в каждом три настоящих503 и один медленный200.
+Блокировки3.5s/1.2s проверяются через granted pg_locks перед HTTP, status и duration проверяются assertions.
+У каждого режима получено9ошибок и3медленных успешных запроса.
+
+### Измеренные результаты
+
+Всего12основных прогонов,7200/7200 подтверждённых запросов без ошибок,360/360 изображений ready.
+Ниже медиана трёх прогонов; ranges и каждый raw result доступны в JSON, точки/отрезки на графике.
+CPU — накопленные секунды двух API или worker за измерительное окно с drain, не проценты host CPU.
+
+| Режим | HTTP p95, ms | p99, ms | API CPU, s | Worker CPU, s | Collector CPU, s | Принятые spans | Collector RX, KiB |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| off | 9.818 | 15.301 | 2.371 | 5.464 | 0.054 | 0 | 13.2 |
+| full | 9.695 | 14.553 | 2.799 | 5.533 | 0.194 | 1950 | 1067.0 |
+| 10% | 9.149 | 13.731 | 2.541 | 5.270 | 0.167 | 176 | 720.9 |
+| 25% | 10.557 | 15.699 | 2.618 | 5.220 | 0.183 | 467 | 802.8 |
+
+| Режим | Сумма sampled anon peaks двух API, MiB | Collector anon peak, MiB | Logs / metric points, медиана |
+| --- | ---: | ---: | ---: |
+| off | 159.60 | 56.02 | 0 / 0 |
+| full | 147.90 | 56.03 | 840 / 1011 |
+| 10% | 168.66 | 58.78 | 840 / 987 |
+| 25% | 147.82 | 58.72 | 840 / 1011 |
+
+Сумма отдельных sampled peaks не является одновременным peak всей группы; интервалы memory/p95 перекрываются.
+Результаты не доказывают ускорение запросов или экономию RAM. Разница CPU небольшая и подвержена шуму.
+Full даёт примерно18% больше API CPU, чем off, в медианах этого опыта.
+25% уменьшает принятые spans на76.1%, сетевой RX Collector на24.8% и API CPU примерно на6.5% относительно full.
+Ingest логов/метрик сохраняется независимо от trace sampling, поэтому суммарный трафик не падает на75%.
+Head sampling применяется к целой цепочке: число дочерних spans в sampled image jobs меняет фактическую долю spans.
+После drain sampled spans SDK и accepted spans Collector совпали во всех12прогонах:
+дополнительная потеря между SDK и Collector в данном измерении0; несэмплированные spans в это число не входят.
+
+| Режим | Сохранён trace ошибок503 | Сохранён trace медленных200 |
+| --- | ---: | ---: |
+| full | 9/9 | 3/3 |
+| 10% | 0/9 | 1/3 |
+| 25% | 1/9 | 1/3 |
+
+Это наблюдения малой выборки, а не оценка вероятности или гарантия. Логи этих HTTP и полные метрики
+сохраняются; sampled проверяется в коррелированном HTTP log;
+для всех36probes также проверено реальное наличие sampled trace и404 для unsampled через Tempo API. Head sampling принимает решение до результата,
+поэтому не может гарантировать trace ошибки. Для расследования воспроизводимого сбоя временно выбирать1.0
+или отправлять контролируемый sampled W3C parent (как proof); это не автоматическое сохранение всех ошибок.
+
+По умолчанию выбран25% как компромисс объёма трасс и покрытия при сохранённых logs/metrics.
+10% даёт ещё меньший ingest, но более редкие полные причинные цепочки. В режиме incident диагностики подходит full.
+Отдельный повтор выбранного режима:600/600 запросов,30/30 изображений ready,
+p95=10.587ms,p99=17.222ms,456accepted spans, дополнительная потеря SDK→Collector0.
+Результат — evidence/balanced-confirmation.json.
+Сырые результаты: [observability-cost.json](evidence/observability-cost.json),
+[cost-summary.json](evidence/cost-summary.json), [sampling-retention.json](evidence/sampling-retention.json).
+
+
+
+## Проверки
+
+Локально выполнено:
+- make check и полная OpenAPI3.1 validation — PASS, оригинальные contracts/tests без изменений.
+- make test и make test-student — public smoke и18тестов API/изображений PASS.
+- make test-observability — реальная causal trace, retry, correlated logs, low-cardinality и canary checks PASS.
+- make test-incidents — оба алерта firing/resolved, trace SQL lock и восстановление PASS.
+- make test-telemetry-outage — бизнес-записи/worker при отказе Tempo, bounded memory, новые traces после возврата PASS.
+- make test-persistence ACTION_SCRIPT=scripts/restart.sh — confirmed business data и старый Bearer сохранены;
+ дополнительный persistence_telemetry.py подтвердил прежние spans/logs и historical Prometheus sample.
+- make test-failover ACTION_SCRIPT=scripts/stop-one.sh —120записей,1временная ошибка,
+ первый успех после отказа2.602s; API восстановлен.
+- student-tests/security.py — anonymous object/list403, strict envelope PASS.
+- student-tests/pipeline_checks.py —13групп инвариантов, реальные exit77, lease recovery/version fencing/replay/cleanup PASS.
+- make research —12прогонов; отдельные sampling_retention.py —9синхронизированных прогонов.
+- promtool check rules —3правила PASS.
+- Grafana UI: dashboard с реальными рядами, Tempo→Loki по тому же trace_id и Loki TraceID→Tempo проверены.
+ Скриншоты evidence/grafana.jpg, trace-ui.jpg, correlated-ui.jpg.
+
+Настоящий отдельный clone commit0e78b2b с новыми secrets и пустыми volumes:
+сборка, миграции, public smoke,18тестов, causal/retry proof и полный restart с прежними
+business data/Bearer/Tempo/Loki/Prometheus history — PASS за83.54s.
+Тестовый Compose project mtusi-lab6-clean-6df218fa и только его volumes удалены;
+основной mtusi-lab6 сохранён. Evidence: clean-clone.json / clean-clone.txt.
+Изменения после clone ограничены отчётом, evidence и дополнительной проверкой исследования, бизнес-код не менялся.
+
+Команды для повторения fault/research проверок запускать последовательно на отдельном учебном стенде:
+они управляют только текущим Compose project и временно останавливают его компоненты.
+CI application workflow добавлен; результат hosted CI не подтверждён, требующее maintainer approval выполнение не выдаётся за PASS.
+Сохранённые txt/JSON — фактические локальные результаты. Raw telemetry содержит только синтетические технические ID.
+
+## Ограничения
+
+Один host, один PostgreSQL/S3/broker/LB/Collector/backend каждого типа остаются SPOF.
+Доказывается отказ API-процесса, перезапуск с сохранением volumes и ограниченный отказ telemetry backend,
+а не потеря диска/питания, distributed HA или exactly-once.
+Миграции и outbox определяют корректность данных независимо от наличия telemetry.
+Логи и метрики тоже имеют bounded asynchronous exporters; при перегрузке/отказе Collector возможны их потери.
+Head sampling не гарантирует сохранение ошибки или редкого хвоста; для этого потребовалась бы отдельная политика tail sampling
+с доказанным буфером/памятью, которую мы не заявляем реализованной.
+Retention24h и file-based stores рассчитаны на локальный synthetic стенд, не промышленное хранение.
+
+## Источники и использование ИИ
+
+Собственная lab5, skeleton/contracts/public tests lab6.
+Официальные материалы:
+- https://opentelemetry.io/docs/languages/python/instrumentation/
+- https://opentelemetry.io/docs/collector/configuration/
+- https://opentelemetry.io/docs/concepts/context-propagation/
+- https://grafana.com/docs/loki/latest/send-data/otel/
+- https://grafana.com/docs/grafana/latest/administration/provisioning/
+- https://prometheus.io/docs/practices/histograms/
+
+Codex использован для реализации, проверки, controlled incidents, исследования и отчёта.
+Численные результаты относятся к сохранённым локальным прогонам. Hosted CI не объявляется прошедшим без отдельного подтверждения.
+\nНастройка Viewer/Explore: https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#viewers_can_edit\n
\ No newline at end of file
diff --git a/compose.yaml b/compose.yaml
new file mode 100644
index 0000000..39a86ba
--- /dev/null
+++ b/compose.yaml
@@ -0,0 +1,298 @@
+name: mtusi-lab6
+x-application: &application
+ image: mtusi-messenger:lab6
+ networks: [backend]
+ environment: &environment
+ DATABASE_URL_FILE: /run/secrets/database_url
+ CACHE_URL_FILE: /run/secrets/cache_url
+ SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
+ HTTP_WORKERS: ${HTTP_WORKERS:-16}
+ DB_POOL_MAX: ${DB_POOL_MAX:-8}
+ DB_ACCESS_MODE: ${DB_ACCESS_MODE:-optimized}
+ LOG_LEVEL: WARNING
+ TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
+ TRACE_SAMPLE_RATIO: ${TRACE_SAMPLE_RATIO:-0.25}
+ OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+ MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100}
+ JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
+ S3_ENDPOINT: http://s3:8333
+ S3_BUCKET: messenger-attachments
+ S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_key
+ S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret
+ secrets: [database_url, cache_url, s3_api_key, s3_api_secret]
+ tmpfs: ["/tmp:size=128m,mode=1777"]
+ mem_limit: ${API_MEMORY:-512m}
+ cpus: ${API_CPUS:-1}
+ stop_grace_period: 10s
+services:
+ migrate:
+ <<: *application
+ command: [python, src/migrate.py]
+ healthcheck:
+ disable: true
+ deploy:
+ replicas: 1
+ depends_on:
+ postgres:
+ condition: service_healthy
+ api:
+ <<: *application
+ build: .
+ deploy:
+ replicas: ${API_REPLICAS:-2}
+ depends_on:
+ otel-collector:
+ condition: service_started
+ migrate:
+ condition: service_completed_successfully
+ redis:
+ condition: service_healthy
+ bucket-bootstrap:
+ condition: service_completed_successfully
+ bucket-bootstrap:
+ <<: *application
+ command: [python, src/bucket_bootstrap.py]
+ environment:
+ <<: *environment
+ S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key
+ S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret
+ secrets: [s3_admin_key, s3_admin_secret]
+ healthcheck:
+ disable: true
+ depends_on:
+ s3:
+ condition: service_healthy
+ audit:
+ <<: *application
+ profiles: [tools]
+ environment:
+ <<: *environment
+ BASE_URL: http://lb:8080
+ BROKER_URL_FILE: /run/secrets/broker_url
+ JOB_LEASE_SECONDS: 5
+ secrets: [database_url, cache_url, s3_api_key, s3_api_secret, s3_admin_key, s3_admin_secret, s3_worker_key, s3_worker_secret, broker_url]
+ healthcheck:
+ disable: true
+ deploy:
+ replicas: 1
+ dispatcher:
+ <<: *application
+ command: [python, src/pipeline.py, dispatch]
+ environment:
+ <<: *environment
+ BROKER_URL_FILE: /run/secrets/broker_url
+ JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60}
+ secrets: [database_url, broker_url]
+ healthcheck:
+ disable: true
+ depends_on:
+ migrate:
+ condition: service_completed_successfully
+ broker:
+ condition: service_healthy
+ restart: unless-stopped
+ worker:
+ <<: *application
+ command: [python, src/pipeline.py, work]
+ environment:
+ <<: *environment
+ BROKER_URL_FILE: /run/secrets/broker_url
+ S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_key
+ S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret
+ JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60}
+ JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
+ WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
+ secrets: [database_url, broker_url, s3_worker_key, s3_worker_secret]
+ mem_limit: ${WORKER_MEMORY:-768m}
+ tmpfs: ["/tmp:size=128m,mode=1777"]
+ cpus: 1
+ deploy:
+ replicas: 1
+ healthcheck:
+ disable: true
+ depends_on:
+ migrate:
+ condition: service_completed_successfully
+ bucket-bootstrap:
+ condition: service_completed_successfully
+ broker:
+ condition: service_healthy
+ restart: unless-stopped
+ broker:
+ image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47
+ command: [redis-server, --appendonly, "yes", --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction, --save, ""]
+ volumes: [broker_data:/data]
+ networks: [backend]
+ mem_limit: 192m
+ cpus: 0.5
+ healthcheck:
+ test: [CMD, redis-cli, ping]
+ interval: 2s
+ timeout: 1s
+ retries: 30
+ s3-store:
+ image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8
+ command: [mini, -dir=/data, -ip=s3-store, -s3=false, -admin.ui=false, -webdav=false, -master.telemetry=false, -master.volumeSizeLimitMB=128, -volume.max=16, -volume.fileSizeLimitMB=11, -volume.concurrentUploadLimitMB=64, -volume.concurrentDownloadLimitMB=64]
+ networks: [storage]
+ volumes: [s3_data:/data]
+ mem_limit: 512m
+ cpus: 1
+ stop_grace_period: 20s
+ healthcheck:
+ test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8888/]
+ interval: 2s
+ timeout: 2s
+ retries: 45
+ s3:
+ image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8
+ command: [s3, -filer=s3-store:8888, -port=8333, -config=/run/secrets/s3_config, -iam=false, -port.iceberg=0, -port.lance=0, -concurrentFileUploadLimit=16, -concurrentUploadLimitMB=32, -readerCacheSizeMB=16]
+ secrets: [s3_config]
+ networks: [backend, storage]
+ mem_limit: 512m
+ environment:
+ GOMEMLIMIT: 384MiB
+ cpus: 0.5
+ depends_on:
+ s3-store:
+ condition: service_healthy
+ healthcheck:
+ test: [CMD-SHELL, 'nc -z 127.0.0.1 8333']
+ interval: 2s
+ timeout: 1s
+ retries: 30
+ lb:
+ image: haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949
+ ports: ["127.0.0.1:${APP_PORT:-18093}:8080"]
+ volumes:
+ - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+ networks: [edge, backend]
+ depends_on:
+ api:
+ condition: service_healthy
+ mem_limit: 64m
+ cpus: 0.5
+ stop_grace_period: 12s
+ healthcheck:
+ test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
+ interval: 2s
+ timeout: 2s
+ retries: 10
+ postgres:
+ image: postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3
+ environment:
+ POSTGRES_USER: messenger
+ POSTGRES_DB: messenger
+ POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
+ secrets: [postgres_password]
+ volumes: [postgres_data:/var/lib/postgresql/data]
+ networks: [backend]
+ healthcheck:
+ test: [CMD, pg_isready, -h, 127.0.0.1, -U, messenger, -d, messenger]
+ interval: 2s
+ timeout: 1s
+ retries: 30
+ mem_limit: 512m
+ cpus: ${POSTGRES_CPUS:-1}
+ redis:
+ image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47
+ command: [redis-server, --maxmemory, 32mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"]
+ networks: [backend]
+ healthcheck:
+ test: [CMD, redis-cli, ping]
+ interval: 2s
+ timeout: 1s
+ retries: 30
+ mem_limit: 64m
+ cpus: 0.25
+ otel-collector:
+ image: otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af
+ command: [--config=/etc/otel/config.yaml]
+ volumes: [./infra/otel-collector.yaml:/etc/otel/config.yaml:ro]
+ networks: [backend, edge]
+ ports: ["127.0.0.1:${COLLECTOR_PORT:-18896}:8888"]
+ mem_limit: 256m
+ cpus: 0.5
+ environment:
+ GOMEMLIMIT: 192MiB
+ prometheus:
+ image: prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996
+ command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.path=/prometheus, --storage.tsdb.retention.time=24h, --storage.tsdb.retention.size=256MB]
+ volumes: [./infra/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro, ./infra/alerts.yaml:/etc/prometheus/alerts.yaml:ro, metrics_data:/prometheus]
+ networks: [backend, edge]
+ ports: ["127.0.0.1:${PROMETHEUS_PORT:-19096}:9090"]
+ mem_limit: 256m
+ cpus: 0.5
+ loki:
+ image: grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd
+ command: [-config.file=/etc/loki/config.yaml]
+ volumes: [./infra/loki.yaml:/etc/loki/config.yaml:ro, logs_data:/loki]
+ networks: [backend, edge]
+ ports: ["127.0.0.1:${LOKI_PORT:-13106}:3100"]
+ mem_limit: 256m
+ cpus: 0.5
+ environment:
+ GOMEMLIMIT: 192MiB
+ tempo:
+ image: grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7
+ command: [-config.file=/etc/tempo/config.yaml]
+ volumes: [./infra/tempo.yaml:/etc/tempo/config.yaml:ro, traces_data:/var/tempo]
+ networks: [backend, edge]
+ ports: ["127.0.0.1:${TEMPO_PORT:-13206}:3200"]
+ mem_limit: 512m
+ cpus: 0.5
+ environment:
+ GOMEMLIMIT: 384MiB
+ grafana:
+ image: grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d
+ volumes: [./infra/grafana/provisioning:/etc/grafana/provisioning:ro, ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro, grafana_data:/var/lib/grafana]
+ networks: [backend, edge]
+ ports: ["127.0.0.1:${GRAFANA_PORT:-13006}:3000"]
+ environment:
+ GF_AUTH_ANONYMOUS_ENABLED: "true"
+ GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
+ GF_AUTH_DISABLE_LOGIN_FORM: "true"
+ GF_AUTH_BASIC_ENABLED: "false"
+ GF_USERS_ALLOW_SIGN_UP: "false"
+ GF_USERS_VIEWERS_CAN_EDIT: "true"
+ GF_ANALYTICS_REPORTING_ENABLED: "false"
+ GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
+ mem_limit: 256m
+ cpus: 0.5
+
+volumes:
+ metrics_data:
+ logs_data:
+ traces_data:
+ grafana_data:
+ postgres_data:
+ broker_data:
+ s3_data:
+networks:
+ edge:
+ storage:
+ internal: true
+ backend:
+ internal: true
+secrets:
+ postgres_password:
+ file: ./secrets/postgres_password
+ database_url:
+ file: ./secrets/database_url
+ cache_url:
+ file: ./secrets/cache_url
+ s3_admin_key:
+ file: ./secrets/s3_admin_key
+ s3_admin_secret:
+ file: ./secrets/s3_admin_secret
+ s3_api_key:
+ file: ./secrets/s3_api_key
+ s3_api_secret:
+ file: ./secrets/s3_api_secret
+ s3_config:
+ file: ./secrets/s3_config
+ broker_url:
+ file: ./secrets/broker_url
+ s3_worker_key:
+ file: ./secrets/s3_worker_key
+ s3_worker_secret:
+ file: ./secrets/s3_worker_secret
diff --git a/evidence/backend-outage.json b/evidence/backend-outage.json
new file mode 100644
index 0000000..6c0467a
--- /dev/null
+++ b/evidence/backend-outage.json
@@ -0,0 +1,100 @@
+{
+ "duration_seconds": 56.09580111503601,
+ "confirmed_messages": 135,
+ "confirmed_attachments_ready": 5,
+ "all_confirmed_messages_retained": true,
+ "collector_samples": [
+ {
+ "elapsed": 2.1163909435272217,
+ "memory": "58.21MiB / 256MiB",
+ "cpu": "1.00%"
+ },
+ {
+ "elapsed": 14.031727075576782,
+ "memory": "58.21MiB / 256MiB",
+ "cpu": "0.59%"
+ },
+ {
+ "elapsed": 25.931473970413208,
+ "memory": "58.77MiB / 256MiB",
+ "cpu": "0.62%"
+ },
+ {
+ "elapsed": 37.78569412231445,
+ "memory": "62.01MiB / 256MiB",
+ "cpu": "0.69%"
+ },
+ {
+ "elapsed": 49.66645312309265,
+ "memory": "62.05MiB / 256MiB",
+ "cpu": "1.13%"
+ }
+ ],
+ "before": {
+ "otelcol_exporter_queue_capacity": 256.0,
+ "otelcol_exporter_queue_size": 0.0,
+ "otelcol_exporter_send_failed_log_records_total": 0.0,
+ "otelcol_exporter_send_failed_metric_points_total": 0.0,
+ "otelcol_exporter_send_failed_spans_total": 0.0,
+ "otelcol_exporter_sent_log_records_total": 824.0,
+ "otelcol_exporter_sent_metric_points_total": 16842.0,
+ "otelcol_exporter_sent_spans_total": 1783.0,
+ "otelcol_process_cpu_seconds_total": 2.12,
+ "otelcol_process_memory_rss_bytes": 203309056.0,
+ "otelcol_process_runtime_heap_alloc_bytes": 32249488.0,
+ "otelcol_process_runtime_total_alloc_bytes_total": 333861280.0,
+ "otelcol_process_runtime_total_sys_memory_bytes": 62215432.0,
+ "otelcol_process_uptime_seconds_total": 401.657121304,
+ "otelcol_processor_accepted_log_records_total": 824.0,
+ "otelcol_processor_accepted_metric_points_total": 16917.0,
+ "otelcol_processor_accepted_spans_total": 1783.0,
+ "otelcol_processor_batch_batch_send_size_bucket": 10026.0,
+ "otelcol_processor_batch_batch_send_size_sum": 19449.0,
+ "otelcol_processor_batch_batch_send_size_count": 478.0,
+ "otelcol_processor_batch_batch_size_trigger_send_total": 11.0,
+ "otelcol_processor_batch_metadata_cardinality": 3.0,
+ "otelcol_processor_batch_timeout_trigger_send_total": 467.0,
+ "otelcol_processor_incoming_items_total": 19524.0,
+ "otelcol_processor_outgoing_items_total": 19524.0,
+ "otelcol_receiver_accepted_log_records_total": 824.0,
+ "otelcol_receiver_accepted_metric_points_total": 16917.0,
+ "otelcol_receiver_accepted_spans_total": 1783.0,
+ "otelcol_receiver_refused_log_records_total": 0.0,
+ "otelcol_receiver_refused_metric_points_total": 0.0,
+ "otelcol_receiver_refused_spans_total": 0.0
+ },
+ "during": {
+ "otelcol_exporter_queue_capacity": 256.0,
+ "otelcol_exporter_queue_size": 37.0,
+ "otelcol_exporter_send_failed_log_records_total": 0.0,
+ "otelcol_exporter_send_failed_metric_points_total": 0.0,
+ "otelcol_exporter_send_failed_spans_total": 156.0,
+ "otelcol_exporter_sent_log_records_total": 1021.0,
+ "otelcol_exporter_sent_metric_points_total": 19388.0,
+ "otelcol_exporter_sent_spans_total": 1801.0,
+ "otelcol_process_cpu_seconds_total": 2.54,
+ "otelcol_process_memory_rss_bytes": 206245888.0,
+ "otelcol_process_runtime_heap_alloc_bytes": 41796440.0,
+ "otelcol_process_runtime_total_alloc_bytes_total": 385908760.0,
+ "otelcol_process_runtime_total_sys_memory_bytes": 66409736.0,
+ "otelcol_process_uptime_seconds_total": 456.688764525,
+ "otelcol_processor_accepted_log_records_total": 1021.0,
+ "otelcol_processor_accepted_metric_points_total": 19388.0,
+ "otelcol_processor_accepted_spans_total": 2356.0,
+ "otelcol_processor_batch_batch_send_size_bucket": 12845.0,
+ "otelcol_processor_batch_batch_send_size_sum": 22765.0,
+ "otelcol_processor_batch_batch_send_size_count": 606.0,
+ "otelcol_processor_batch_batch_size_trigger_send_total": 11.0,
+ "otelcol_processor_batch_metadata_cardinality": 3.0,
+ "otelcol_processor_batch_timeout_trigger_send_total": 595.0,
+ "otelcol_processor_incoming_items_total": 22765.0,
+ "otelcol_processor_outgoing_items_total": 22765.0,
+ "otelcol_receiver_accepted_log_records_total": 1021.0,
+ "otelcol_receiver_accepted_metric_points_total": 19388.0,
+ "otelcol_receiver_accepted_spans_total": 2356.0,
+ "otelcol_receiver_refused_log_records_total": 0.0,
+ "otelcol_receiver_refused_metric_points_total": 0.0,
+ "otelcol_receiver_refused_spans_total": 0.0
+ },
+ "recovered_trace_id": "1555228135a04659bac5654623c7a552"
+}
diff --git a/evidence/balanced-confirmation.json b/evidence/balanced-confirmation.json
new file mode 100644
index 0000000..18e5b8b
--- /dev/null
+++ b/evidence/balanced-confirmation.json
@@ -0,0 +1,141 @@
+{
+ "mode": "balanced25",
+ "enabled": 1,
+ "sample_ratio": 0.25,
+ "repeat": 0,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.970493208,
+ "successful_rps": 30.044325583288316,
+ "p50_seconds": 0.003056625000001034,
+ "p95_seconds": 0.010586957999999314,
+ "p99_seconds": 0.017221916999998754,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.6645160000000008,
+ "peak_sampled_anonymous_bytes": 197382144,
+ "peak_sampled_working_set_bytes": 206495744,
+ "rx_bytes": 4570530
+ },
+ "broker": {
+ "cpu_seconds": 0.15844099999999983,
+ "peak_sampled_anonymous_bytes": 3289088,
+ "peak_sampled_working_set_bytes": 3899392,
+ "rx_bytes": 46110
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.7753760000000005,
+ "peak_sampled_anonymous_bytes": 45830144,
+ "peak_sampled_working_set_bytes": 48340992,
+ "rx_bytes": 472693
+ },
+ "grafana": {
+ "cpu_seconds": 0.34897599999999995,
+ "peak_sampled_anonymous_bytes": 106962944,
+ "peak_sampled_working_set_bytes": 114311168,
+ "rx_bytes": 77173
+ },
+ "lb": {
+ "cpu_seconds": 0.24454800000000043,
+ "peak_sampled_anonymous_bytes": 6942720,
+ "peak_sampled_working_set_bytes": 7999488,
+ "rx_bytes": 4287798
+ },
+ "loki": {
+ "cpu_seconds": 0.16779699999999975,
+ "peak_sampled_anonymous_bytes": 108310528,
+ "peak_sampled_working_set_bytes": 112246784,
+ "rx_bytes": 111200
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.19291599999999987,
+ "peak_sampled_anonymous_bytes": 57331712,
+ "peak_sampled_working_set_bytes": 63811584,
+ "rx_bytes": 863090
+ },
+ "postgres": {
+ "cpu_seconds": 1.631062,
+ "peak_sampled_anonymous_bytes": 15544320,
+ "peak_sampled_working_set_bytes": 38952960,
+ "rx_bytes": 1139116
+ },
+ "prometheus": {
+ "cpu_seconds": 0.1952210000000003,
+ "peak_sampled_anonymous_bytes": 56045568,
+ "peak_sampled_working_set_bytes": 59535360,
+ "rx_bytes": 116869
+ },
+ "redis": {
+ "cpu_seconds": 0.17422799999999983,
+ "peak_sampled_anonymous_bytes": 3067904,
+ "peak_sampled_working_set_bytes": 3612672,
+ "rx_bytes": 119742
+ },
+ "s3": {
+ "cpu_seconds": 0.4350659999999995,
+ "peak_sampled_anonymous_bytes": 69742592,
+ "peak_sampled_working_set_bytes": 71454720,
+ "rx_bytes": 11144479
+ },
+ "s3-store": {
+ "cpu_seconds": 0.9894250000000007,
+ "peak_sampled_anonymous_bytes": 134537216,
+ "peak_sampled_working_set_bytes": 137109504,
+ "rx_bytes": 7486398
+ },
+ "tempo": {
+ "cpu_seconds": 0.11411999999999978,
+ "peak_sampled_anonymous_bytes": 360697856,
+ "peak_sampled_working_set_bytes": 364331008,
+ "rx_bytes": 37382
+ },
+ "worker": {
+ "cpu_seconds": 5.430313999999999,
+ "peak_sampled_anonymous_bytes": 77266944,
+ "peak_sampled_working_set_bytes": 82108416,
+ "rx_bytes": 3761105
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 456.0,
+ "otelcol_receiver_accepted_log_records_total": 840.0,
+ "otelcol_receiver_accepted_metric_points_total": 1154.0,
+ "otelcol_exporter_sent_spans_total": 456.0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 456.0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "0c11495cc94443caadcd12bd17a86dd0",
+ "status": 503,
+ "latency_seconds": 1.4905514579999988,
+ "trace_sampled": true,
+ "trace_id": "aeacd5bdeb7d0f073b2a2cc44487c22c"
+ },
+ {
+ "request_id": "b0871c2ae52245ce882338cf01bd6a4d",
+ "status": 503,
+ "latency_seconds": 1.4968927090000008,
+ "trace_sampled": false,
+ "trace_id": "4d8c492fae68617ec5dbd2bc849de2e9"
+ },
+ {
+ "request_id": "d1d60c7ee15f498b988e031e15dc7d27",
+ "status": 503,
+ "latency_seconds": 1.4817318749999941,
+ "trace_sampled": false,
+ "trace_id": "931e742a91866246ac3b77dd062517c6"
+ },
+ {
+ "request_id": "95e79f3e1894491c9355c15e0e79a695",
+ "status": 200,
+ "latency_seconds": 1.1869852499999993,
+ "trace_sampled": false,
+ "trace_id": "34e1ad7e9c366b666caf53045be238e9"
+ }
+ ]
+}
diff --git a/evidence/clean-clone.json b/evidence/clean-clone.json
new file mode 100644
index 0000000..d2b5915
--- /dev/null
+++ b/evidence/clean-clone.json
@@ -0,0 +1,11 @@
+{
+ "source_commit": "0e78b2b27de93464a6bd0974026bee4bfe4f602a",
+ "project": "mtusi-lab6-clean-6df218fa",
+ "new_secrets": true,
+ "new_empty_volumes": true,
+ "build_smoke_18_api_tests": true,
+ "real_observability_proof": true,
+ "business_and_telemetry_persistence": true,
+ "temporary_project_removed": true,
+ "seconds": 83.54
+}
diff --git a/evidence/clean-clone.txt b/evidence/clean-clone.txt
new file mode 100644
index 0000000..efb144e
--- /dev/null
+++ b/evidence/clean-clone.txt
@@ -0,0 +1,338 @@
+RUN python3 scripts/bootstrap.py
+Local configuration ready; no secrets printed.
+RUN docker compose build api
+ Image mtusi-messenger:lab6 Building
+#1 [internal] load local bake definitions
+#1 reading from stdin 707B done
+#1 DONE 0.0s
+
+#2 [internal] load build definition from Dockerfile
+#2 transferring dockerfile: 739B done
+#2 DONE 0.0s
+
+#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
+#3 DONE 1.0s
+
+#4 [internal] load .dockerignore
+#4 transferring context: 203B done
+#4 DONE 0.0s
+
+#5 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
+#5 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done
+#5 DONE 0.0s
+
+#6 [internal] load build context
+#6 transferring context: 88.54kB done
+#6 DONE 0.0s
+
+#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
+#7 CACHED
+
+#8 [5/7] RUN pip install --no-cache-dir -r requirements.txt
+#8 CACHED
+
+#9 [6/7] COPY src/ ./src/
+#9 CACHED
+
+#10 [2/7] WORKDIR /app
+#10 CACHED
+
+#11 [4/7] COPY requirements.txt ./
+#11 CACHED
+
+#12 [7/7] COPY migrations/ ./migrations/
+#12 CACHED
+
+#13 exporting to image
+#13 exporting layers done
+#13 exporting manifest sha256:e533c95082fcc2d03df488d24c04c74c3205f96ebce7d39fcf544e15ed2ac2c9 done
+#13 exporting config sha256:d5159a690cc9eed35b1e1df5510f9f8595fbb5e303c09bde83ea92197bbfab8c done
+#13 exporting attestation manifest sha256:91fc899a62ccb18479e1135cbfe8c85ae865353f7315c7c8897a6418fe283c11 done
+#13 exporting manifest list sha256:a0c792a12285d3f4914e1ce37a8832777d639abd2653a71ff0b581b28f2f5e6b done
+#13 naming to docker.io/library/mtusi-messenger:lab6 done
+#13 unpacking to docker.io/library/mtusi-messenger:lab6 done
+#13 DONE 0.0s
+
+#14 resolving provenance for metadata file
+#14 DONE 0.0s
+ Image mtusi-messenger:lab6 Built
+RUN docker compose up -d --wait --wait-timeout 180
+ Network mtusi-lab6-clean-6df218fa_edge Creating
+ Network mtusi-lab6-clean-6df218fa_edge Created
+ Network mtusi-lab6-clean-6df218fa_backend Creating
+ Network mtusi-lab6-clean-6df218fa_backend Created
+ Network mtusi-lab6-clean-6df218fa_storage Creating
+ Network mtusi-lab6-clean-6df218fa_storage Created
+ Volume mtusi-lab6-clean-6df218fa_logs_data Creating
+ Volume mtusi-lab6-clean-6df218fa_logs_data Created
+ Volume mtusi-lab6-clean-6df218fa_metrics_data Creating
+ Volume mtusi-lab6-clean-6df218fa_metrics_data Created
+ Volume mtusi-lab6-clean-6df218fa_traces_data Creating
+ Volume mtusi-lab6-clean-6df218fa_traces_data Created
+ Volume mtusi-lab6-clean-6df218fa_postgres_data Creating
+ Volume mtusi-lab6-clean-6df218fa_postgres_data Created
+ Volume mtusi-lab6-clean-6df218fa_s3_data Creating
+ Volume mtusi-lab6-clean-6df218fa_s3_data Created
+ Volume mtusi-lab6-clean-6df218fa_broker_data Creating
+ Volume mtusi-lab6-clean-6df218fa_broker_data Created
+ Volume mtusi-lab6-clean-6df218fa_grafana_data Creating
+ Volume mtusi-lab6-clean-6df218fa_grafana_data Created
+ Container mtusi-lab6-clean-6df218fa-grafana-1 Creating
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Creating
+ Container mtusi-lab6-clean-6df218fa-prometheus-1 Creating
+ Container mtusi-lab6-clean-6df218fa-otel-collector-1 Creating
+ Container mtusi-lab6-clean-6df218fa-loki-1 Creating
+ Container mtusi-lab6-clean-6df218fa-redis-1 Creating
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Creating
+ Container mtusi-lab6-clean-6df218fa-broker-1 Creating
+ Container mtusi-lab6-clean-6df218fa-tempo-1 Creating
+ Container mtusi-lab6-clean-6df218fa-otel-collector-1 Created
+ Container mtusi-lab6-clean-6df218fa-grafana-1 Created
+ Container mtusi-lab6-clean-6df218fa-prometheus-1 Created
+ Container mtusi-lab6-clean-6df218fa-loki-1 Created
+ Container mtusi-lab6-clean-6df218fa-tempo-1 Created
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Created
+ Container mtusi-lab6-clean-6df218fa-broker-1 Created
+ Container mtusi-lab6-clean-6df218fa-s3-1 Creating
+ Container mtusi-lab6-clean-6df218fa-redis-1 Created
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Created
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Creating
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Created
+ Container mtusi-lab6-clean-6df218fa-dispatcher-1 Creating
+ Container mtusi-lab6-clean-6df218fa-s3-1 Created
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Creating
+ Container mtusi-lab6-clean-6df218fa-dispatcher-1 Created
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Created
+ Container mtusi-lab6-clean-6df218fa-worker-1 Creating
+ Container mtusi-lab6-clean-6df218fa-api-1 Creating
+ Container mtusi-lab6-clean-6df218fa-api-2 Creating
+ Container mtusi-lab6-clean-6df218fa-api-1 Created
+ Container mtusi-lab6-clean-6df218fa-api-2 Created
+ Container mtusi-lab6-clean-6df218fa-lb-1 Creating
+ Container mtusi-lab6-clean-6df218fa-worker-1 Created
+ Container mtusi-lab6-clean-6df218fa-lb-1 Created
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Starting
+ Container mtusi-lab6-clean-6df218fa-redis-1 Starting
+ Container mtusi-lab6-clean-6df218fa-tempo-1 Starting
+ Container mtusi-lab6-clean-6df218fa-prometheus-1 Starting
+ Container mtusi-lab6-clean-6df218fa-loki-1 Starting
+ Container mtusi-lab6-clean-6df218fa-broker-1 Starting
+ Container mtusi-lab6-clean-6df218fa-grafana-1 Starting
+ Container mtusi-lab6-clean-6df218fa-otel-collector-1 Starting
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Starting
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Started
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-broker-1 Started
+ Container mtusi-lab6-clean-6df218fa-redis-1 Started
+ Container mtusi-lab6-clean-6df218fa-otel-collector-1 Started
+ Container mtusi-lab6-clean-6df218fa-prometheus-1 Started
+ Container mtusi-lab6-clean-6df218fa-loki-1 Started
+ Container mtusi-lab6-clean-6df218fa-tempo-1 Started
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Started
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-grafana-1 Started
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Starting
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Started
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-broker-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-broker-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
+ Container mtusi-lab6-clean-6df218fa-dispatcher-1 Starting
+ Container mtusi-lab6-clean-6df218fa-dispatcher-1 Started
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-s3-1 Starting
+ Container mtusi-lab6-clean-6df218fa-s3-1 Started
+ Container mtusi-lab6-clean-6df218fa-s3-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-s3-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Starting
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Started
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-redis-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-broker-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-broker-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-redis-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited
+ Container mtusi-lab6-clean-6df218fa-worker-1 Starting
+ Container mtusi-lab6-clean-6df218fa-api-1 Starting
+ Container mtusi-lab6-clean-6df218fa-worker-1 Started
+ Container mtusi-lab6-clean-6df218fa-api-1 Started
+ Container mtusi-lab6-clean-6df218fa-api-2 Starting
+ Container mtusi-lab6-clean-6df218fa-api-2 Started
+ Container mtusi-lab6-clean-6df218fa-api-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-api-2 Waiting
+ Container mtusi-lab6-clean-6df218fa-api-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-api-2 Healthy
+ Container mtusi-lab6-clean-6df218fa-lb-1 Starting
+ Container mtusi-lab6-clean-6df218fa-lb-1 Started
+ Container mtusi-lab6-clean-6df218fa-broker-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-worker-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-prometheus-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-lb-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-loki-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-tempo-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-dispatcher-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-otel-collector-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-grafana-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-redis-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-s3-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-api-1 Waiting
+ Container mtusi-lab6-clean-6df218fa-api-2 Waiting
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-loki-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
+ Container mtusi-lab6-clean-6df218fa-broker-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-worker-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-tempo-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-otel-collector-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-grafana-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-redis-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-dispatcher-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-s3-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-prometheus-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-api-1 Healthy
+ Container mtusi-lab6-clean-6df218fa-api-2 Healthy
+ Container mtusi-lab6-clean-6df218fa-lb-1 Healthy
+RUN python3 scripts/wait_ready.py
+Ready through load balancer.
+RUN make check test test-student BASE_URL=http://127.0.0.1:18094
+python3 scripts/check_contract.py
+PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+python3 tests/smoke.py --base-url "http://127.0.0.1:18094"
+PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+BASE_URL="http://127.0.0.1:18094" python3 -m unittest discover -s student-tests -p 'test_*.py' -v
+test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
+test_cursor_scope_and_validation (test_api.APIContract) ... ok
+test_foreign_chat (test_api.APIContract) ... ok
+test_idempotency_concurrent_replicas (test_api.APIContract) ... ok
+test_idempotency_conflicting_race (test_api.APIContract) ... ok
+test_idempotency_invalid_keys (test_api.APIContract) ... ok
+test_idempotency_scope (test_api.APIContract) ... ok
+test_idempotency_semantic_body (test_api.APIContract) ... ok
+test_invalid_chat_creation (test_api.APIContract) ... ok
+test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
+test_protocol_error_headers (test_api.APIContract) ... ok
+test_registration_race (test_api.APIContract) ... ok
+test_unicode_limits (test_api.APIContract) ... ok
+test_unicode_nul_round_trip (test_api.APIContract) ... ok
+test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
+test_async_variants_and_replay (test_images.Images) ... ok
+test_mime_bytes_pixels_and_filename (test_images.Images) ... ok
+test_private_and_cross_chat (test_images.Images) ... ok
+
+----------------------------------------------------------------------
+Ran 18 tests in 4.950s
+
+OK
+RUN python3 -u student-tests/observability.py proof
+ Container mtusi-lab6-clean-6df218fa-worker-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-worker-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-s3-1 Paused
+ Container mtusi-lab6-clean-6df218fa-worker-1 Starting
+ Container mtusi-lab6-clean-6df218fa-worker-1 Started
+ Container mtusi-lab6-clean-6df218fa-s3-1 Unpaused
+ Container mtusi-lab6-clean-6df218fa-worker-1 Running
+PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.
+RUN python3 -u student-tests/persistence_telemetry.py
+PASS: full business restart, original Bearer, historical Tempo trace, Loki logs and fixed-time Prometheus sample.
+RUN docker compose down -v --remove-orphans
+ Container mtusi-lab6-clean-6df218fa-loki-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-lb-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-grafana-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-tempo-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-dispatcher-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-prometheus-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-worker-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-dispatcher-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-dispatcher-1 Removing
+ Container mtusi-lab6-clean-6df218fa-dispatcher-1 Removed
+ Container mtusi-lab6-clean-6df218fa-lb-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-lb-1 Removing
+ Container mtusi-lab6-clean-6df218fa-lb-1 Removed
+ Container mtusi-lab6-clean-6df218fa-api-2 Stopping
+ Container mtusi-lab6-clean-6df218fa-api-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-worker-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-worker-1 Removing
+ Container mtusi-lab6-clean-6df218fa-worker-1 Removed
+ Container mtusi-lab6-clean-6df218fa-broker-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-grafana-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-grafana-1 Removing
+ Container mtusi-lab6-clean-6df218fa-prometheus-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-prometheus-1 Removing
+ Container mtusi-lab6-clean-6df218fa-grafana-1 Removed
+ Container mtusi-lab6-clean-6df218fa-prometheus-1 Removed
+ Container mtusi-lab6-clean-6df218fa-broker-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-broker-1 Removing
+ Container mtusi-lab6-clean-6df218fa-broker-1 Removed
+ Container mtusi-lab6-clean-6df218fa-api-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-api-1 Removing
+ Container mtusi-lab6-clean-6df218fa-api-1 Removed
+ Container mtusi-lab6-clean-6df218fa-api-2 Stopped
+ Container mtusi-lab6-clean-6df218fa-api-2 Removing
+ Container mtusi-lab6-clean-6df218fa-api-2 Removed
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-otel-collector-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-redis-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Removing
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Removing
+ Container mtusi-lab6-clean-6df218fa-migrate-1 Removed
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Removed
+ Container mtusi-lab6-clean-6df218fa-s3-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-redis-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-redis-1 Removing
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Removing
+ Container mtusi-lab6-clean-6df218fa-redis-1 Removed
+ Container mtusi-lab6-clean-6df218fa-postgres-1 Removed
+ Container mtusi-lab6-clean-6df218fa-loki-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-loki-1 Removing
+ Container mtusi-lab6-clean-6df218fa-loki-1 Removed
+ Container mtusi-lab6-clean-6df218fa-s3-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-s3-1 Removing
+ Container mtusi-lab6-clean-6df218fa-otel-collector-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-otel-collector-1 Removing
+ Container mtusi-lab6-clean-6df218fa-s3-1 Removed
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Stopping
+ Container mtusi-lab6-clean-6df218fa-otel-collector-1 Removed
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Removing
+ Container mtusi-lab6-clean-6df218fa-s3-store-1 Removed
+ Container mtusi-lab6-clean-6df218fa-tempo-1 Stopped
+ Container mtusi-lab6-clean-6df218fa-tempo-1 Removing
+ Container mtusi-lab6-clean-6df218fa-tempo-1 Removed
+ Volume mtusi-lab6-clean-6df218fa_broker_data Removing
+ Network mtusi-lab6-clean-6df218fa_storage Removing
+ Volume mtusi-lab6-clean-6df218fa_metrics_data Removing
+ Volume mtusi-lab6-clean-6df218fa_grafana_data Removing
+ Volume mtusi-lab6-clean-6df218fa_traces_data Removing
+ Network mtusi-lab6-clean-6df218fa_edge Removing
+ Volume mtusi-lab6-clean-6df218fa_s3_data Removing
+ Volume mtusi-lab6-clean-6df218fa_logs_data Removing
+ Volume mtusi-lab6-clean-6df218fa_postgres_data Removing
+ Network mtusi-lab6-clean-6df218fa_backend Removing
+ Volume mtusi-lab6-clean-6df218fa_broker_data Removed
+ Volume mtusi-lab6-clean-6df218fa_metrics_data Removed
+ Volume mtusi-lab6-clean-6df218fa_logs_data Removed
+ Volume mtusi-lab6-clean-6df218fa_grafana_data Removed
+ Volume mtusi-lab6-clean-6df218fa_traces_data Removed
+ Volume mtusi-lab6-clean-6df218fa_s3_data Removed
+ Volume mtusi-lab6-clean-6df218fa_postgres_data Removed
+ Network mtusi-lab6-clean-6df218fa_storage Removed
+ Network mtusi-lab6-clean-6df218fa_backend Removed
+ Network mtusi-lab6-clean-6df218fa_edge Removed
diff --git a/evidence/correlated-ui.jpg b/evidence/correlated-ui.jpg
new file mode 100644
index 0000000..5b6097e
Binary files /dev/null and b/evidence/correlated-ui.jpg differ
diff --git a/evidence/cost-summary.json b/evidence/cost-summary.json
new file mode 100644
index 0000000..4ac5bab
--- /dev/null
+++ b/evidence/cost-summary.json
@@ -0,0 +1,86 @@
+{
+ "off": {
+ "runs": 3,
+ "requests": 1800,
+ "ready_images": 90,
+ "p95_ms": 9.81791700002077,
+ "p99_ms": 15.300541999977213,
+ "api_cpu_seconds": 2.3706709999999998,
+ "worker_cpu_seconds": 5.463901,
+ "collector_cpu_seconds": 0.053601000000000454,
+ "api_sum_sampled_anon_peak_mib": 159.59765625,
+ "collector_anon_peak_mib": 56.015625,
+ "accepted_spans": 0,
+ "accepted_logs": 0,
+ "accepted_metric_points": 0,
+ "collector_receive_kib": 13.2080078125,
+ "sampled_spans_not_accepted": 0,
+ "error_probe_traces_kept": 0,
+ "error_probes": 1,
+ "slow_probe_traces_kept": 0,
+ "slow_probes": 11
+ },
+ "full": {
+ "runs": 3,
+ "requests": 1800,
+ "ready_images": 90,
+ "p95_ms": 9.695250000049782,
+ "p99_ms": 14.552875000049426,
+ "api_cpu_seconds": 2.7994320000000004,
+ "worker_cpu_seconds": 5.533063,
+ "collector_cpu_seconds": 0.19410300000000014,
+ "api_sum_sampled_anon_peak_mib": 147.90234375,
+ "collector_anon_peak_mib": 56.02734375,
+ "accepted_spans": 1950.0,
+ "accepted_logs": 840.0,
+ "accepted_metric_points": 1011.0,
+ "collector_receive_kib": 1067.025390625,
+ "sampled_spans_not_accepted": 0,
+ "error_probe_traces_kept": 4,
+ "error_probes": 4,
+ "slow_probe_traces_kept": 8,
+ "slow_probes": 8
+ },
+ "sample10": {
+ "runs": 3,
+ "requests": 1800,
+ "ready_images": 90,
+ "p95_ms": 9.149458999957005,
+ "p99_ms": 13.731374999991885,
+ "api_cpu_seconds": 2.540737,
+ "worker_cpu_seconds": 5.2699810000000005,
+ "collector_cpu_seconds": 0.16662400000000055,
+ "api_sum_sampled_anon_peak_mib": 168.65625,
+ "collector_anon_peak_mib": 58.78125,
+ "accepted_spans": 176.0,
+ "accepted_logs": 840.0,
+ "accepted_metric_points": 987.0,
+ "collector_receive_kib": 720.869140625,
+ "sampled_spans_not_accepted": 0,
+ "error_probe_traces_kept": 0,
+ "error_probes": 1,
+ "slow_probe_traces_kept": 0,
+ "slow_probes": 11
+ },
+ "sample25": {
+ "runs": 3,
+ "requests": 1800,
+ "ready_images": 90,
+ "p95_ms": 10.556667000003017,
+ "p99_ms": 15.699083000015435,
+ "api_cpu_seconds": 2.6178429999999997,
+ "worker_cpu_seconds": 5.2202720000000005,
+ "collector_cpu_seconds": 0.18281499999999973,
+ "api_sum_sampled_anon_peak_mib": 147.81640625,
+ "collector_anon_peak_mib": 58.71875,
+ "accepted_spans": 467.0,
+ "accepted_logs": 840.0,
+ "accepted_metric_points": 1011.0,
+ "collector_receive_kib": 802.8349609375,
+ "sampled_spans_not_accepted": 0,
+ "error_probe_traces_kept": 1,
+ "error_probes": 2,
+ "slow_probe_traces_kept": 3,
+ "slow_probes": 10
+ }
+}
diff --git a/evidence/failover.txt b/evidence/failover.txt
new file mode 100644
index 0000000..c537b6a
--- /dev/null
+++ b/evidence/failover.txt
@@ -0,0 +1,4 @@
+python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18093"
+Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
+{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 1, "first_recovery_seconds": 2.602}
+PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
diff --git a/evidence/final-api-tests.txt b/evidence/final-api-tests.txt
new file mode 100644
index 0000000..cc6138f
--- /dev/null
+++ b/evidence/final-api-tests.txt
@@ -0,0 +1,28 @@
+python3 scripts/check_contract.py
+PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+python3 tests/smoke.py --base-url "http://127.0.0.1:18093"
+PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+BASE_URL="http://127.0.0.1:18093" python3 -m unittest discover -s student-tests -p 'test_*.py' -v
+test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
+test_cursor_scope_and_validation (test_api.APIContract) ... ok
+test_foreign_chat (test_api.APIContract) ... ok
+test_idempotency_concurrent_replicas (test_api.APIContract) ... ok
+test_idempotency_conflicting_race (test_api.APIContract) ... ok
+test_idempotency_invalid_keys (test_api.APIContract) ... ok
+test_idempotency_scope (test_api.APIContract) ... ok
+test_idempotency_semantic_body (test_api.APIContract) ... ok
+test_invalid_chat_creation (test_api.APIContract) ... ok
+test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
+test_protocol_error_headers (test_api.APIContract) ... ok
+test_registration_race (test_api.APIContract) ... ok
+test_unicode_limits (test_api.APIContract) ... ok
+test_unicode_nul_round_trip (test_api.APIContract) ... ok
+test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
+test_async_variants_and_replay (test_images.Images) ... ok
+test_mime_bytes_pixels_and_filename (test_images.Images) ... ok
+test_private_and_cross_chat (test_images.Images) ... ok
+
+----------------------------------------------------------------------
+Ran 18 tests in 5.414s
+
+OK
diff --git a/evidence/final-validation.txt b/evidence/final-validation.txt
new file mode 100644
index 0000000..3bc31b6
--- /dev/null
+++ b/evidence/final-validation.txt
@@ -0,0 +1 @@
+PASS: Python syntax; original README/COURSE/CONTRIBUTING/contracts/public tests unchanged; actual local PostgreSQL password and S3 credentials absent from tracked/new files. Non-secret internal Redis service URLs intentionally appear in bootstrap.
diff --git a/evidence/grafana.jpg b/evidence/grafana.jpg
new file mode 100644
index 0000000..983104e
Binary files /dev/null and b/evidence/grafana.jpg differ
diff --git a/evidence/incidents.json b/evidence/incidents.json
new file mode 100644
index 0000000..f7adf34
--- /dev/null
+++ b/evidence/incidents.json
@@ -0,0 +1,7683 @@
+{
+ "worker_stop": {
+ "injected_at": 1795163117.556709,
+ "detected_at": 1795163140.329111,
+ "detection_seconds": 22.772402048110962,
+ "firing": [
+ {
+ "labels": {
+ "alertname": "QueueLag",
+ "severity": "warning"
+ },
+ "annotations": {
+ "runbook": "infra/RUNBOOK.md#queue-lag",
+ "summary": "Accepted unfinished image work older than 10 seconds"
+ },
+ "state": "firing",
+ "activeAt": "2026-11-20T08:25:32.478517494Z",
+ "value": "1.7749819e+01"
+ }
+ ],
+ "metrics": {
+ "queue_depth": [
+ {
+ "metric": {
+ "__name__": "queue_depth",
+ "deployment_environment_name": "lab",
+ "exported_instance": "11a51b423246479e8ca2b43c8ec27a38",
+ "exported_job": "messenger-api",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "11a51b423246479e8ca2b43c8ec27a38",
+ "service_name": "messenger-api",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.428,
+ "0"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "queue_depth",
+ "deployment_environment_name": "lab",
+ "exported_instance": "8233dde3da6d4f13ab70ce40e99fc803",
+ "exported_job": "messenger-api",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "8233dde3da6d4f13ab70ce40e99fc803",
+ "service_name": "messenger-api",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.428,
+ "0"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "queue_depth",
+ "deployment_environment_name": "lab",
+ "exported_instance": "fb68c5cfe0a64787bba2c8036179590b",
+ "exported_job": "messenger-dispatcher",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "fb68c5cfe0a64787bba2c8036179590b",
+ "service_name": "messenger-dispatcher",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.428,
+ "3"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "queue_depth",
+ "deployment_environment_name": "lab",
+ "exported_instance": "c16c37ace1824e80bfe39a6f30807466",
+ "exported_job": "messenger-worker",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "c16c37ace1824e80bfe39a6f30807466",
+ "service_name": "messenger-worker",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.428,
+ "0"
+ ]
+ }
+ ],
+ "queue_oldest_age_seconds": [
+ {
+ "metric": {
+ "__name__": "queue_oldest_age_seconds",
+ "deployment_environment_name": "lab",
+ "exported_instance": "11a51b423246479e8ca2b43c8ec27a38",
+ "exported_job": "messenger-api",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "11a51b423246479e8ca2b43c8ec27a38",
+ "service_name": "messenger-api",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.43,
+ "0"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "queue_oldest_age_seconds",
+ "deployment_environment_name": "lab",
+ "exported_instance": "8233dde3da6d4f13ab70ce40e99fc803",
+ "exported_job": "messenger-api",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "8233dde3da6d4f13ab70ce40e99fc803",
+ "service_name": "messenger-api",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.43,
+ "0"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "queue_oldest_age_seconds",
+ "deployment_environment_name": "lab",
+ "exported_instance": "fb68c5cfe0a64787bba2c8036179590b",
+ "exported_job": "messenger-dispatcher",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "fb68c5cfe0a64787bba2c8036179590b",
+ "service_name": "messenger-dispatcher",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.43,
+ "19.929928"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "queue_oldest_age_seconds",
+ "deployment_environment_name": "lab",
+ "exported_instance": "c16c37ace1824e80bfe39a6f30807466",
+ "exported_job": "messenger-worker",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "c16c37ace1824e80bfe39a6f30807466",
+ "service_name": "messenger-worker",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.43,
+ "0"
+ ]
+ }
+ ],
+ "worker_busy": [
+ {
+ "metric": {
+ "__name__": "worker_busy",
+ "deployment_environment_name": "lab",
+ "exported_instance": "11a51b423246479e8ca2b43c8ec27a38",
+ "exported_job": "messenger-api",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "11a51b423246479e8ca2b43c8ec27a38",
+ "service_name": "messenger-api",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.432,
+ "0"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "worker_busy",
+ "deployment_environment_name": "lab",
+ "exported_instance": "8233dde3da6d4f13ab70ce40e99fc803",
+ "exported_job": "messenger-api",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "8233dde3da6d4f13ab70ce40e99fc803",
+ "service_name": "messenger-api",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.432,
+ "0"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "worker_busy",
+ "deployment_environment_name": "lab",
+ "exported_instance": "fb68c5cfe0a64787bba2c8036179590b",
+ "exported_job": "messenger-dispatcher",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "fb68c5cfe0a64787bba2c8036179590b",
+ "service_name": "messenger-dispatcher",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.432,
+ "0"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "worker_busy",
+ "deployment_environment_name": "lab",
+ "exported_instance": "c16c37ace1824e80bfe39a6f30807466",
+ "exported_job": "messenger-worker",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "c16c37ace1824e80bfe39a6f30807466",
+ "service_name": "messenger-worker",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.432,
+ "0"
+ ]
+ }
+ ],
+ "worker_capacity": [
+ {
+ "metric": {
+ "__name__": "worker_capacity",
+ "deployment_environment_name": "lab",
+ "exported_instance": "11a51b423246479e8ca2b43c8ec27a38",
+ "exported_job": "messenger-api",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "11a51b423246479e8ca2b43c8ec27a38",
+ "service_name": "messenger-api",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.434,
+ "0"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "worker_capacity",
+ "deployment_environment_name": "lab",
+ "exported_instance": "8233dde3da6d4f13ab70ce40e99fc803",
+ "exported_job": "messenger-api",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "8233dde3da6d4f13ab70ce40e99fc803",
+ "service_name": "messenger-api",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.434,
+ "0"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "worker_capacity",
+ "deployment_environment_name": "lab",
+ "exported_instance": "fb68c5cfe0a64787bba2c8036179590b",
+ "exported_job": "messenger-dispatcher",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "fb68c5cfe0a64787bba2c8036179590b",
+ "service_name": "messenger-dispatcher",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.434,
+ "0"
+ ]
+ },
+ {
+ "metric": {
+ "__name__": "worker_capacity",
+ "deployment_environment_name": "lab",
+ "exported_instance": "c16c37ace1824e80bfe39a6f30807466",
+ "exported_job": "messenger-worker",
+ "instance": "otel-collector:8889",
+ "job": "messenger",
+ "otel_scope_name": "messenger",
+ "service_instance_id": "c16c37ace1824e80bfe39a6f30807466",
+ "service_name": "messenger-worker",
+ "service_version": "lab6.1",
+ "telemetry_sdk_language": "python",
+ "telemetry_sdk_name": "opentelemetry",
+ "telemetry_sdk_version": "1.36.0"
+ },
+ "value": [
+ 1795163140.434,
+ "1"
+ ]
+ }
+ ]
+ },
+ "diagnosis": "Backlog and oldest age grow; worker completion stops while API acceptance still succeeds. Gauges expire after 30s.",
+ "resolved_at": 1795163146.0027971
+ },
+ "db_lock": {
+ "injected_at": 1795163147.011447,
+ "detected_at": 1795163160.154182,
+ "detection_seconds": 13.142735004425049,
+ "firing": [
+ {
+ "labels": {
+ "alertname": "APIHighErrorRate",
+ "severity": "warning"
+ },
+ "annotations": {
+ "runbook": "infra/RUNBOOK.md#api-errors",
+ "summary": "API 5xx share exceeds 5 percent in the last 30 seconds"
+ },
+ "state": "firing",
+ "activeAt": "2026-11-20T08:25:52.478517494Z",
+ "value": "7.484207931500946e-01"
+ }
+ ],
+ "error_latencies_seconds": [
+ 1.510284459000001,
+ 1.5103016249999968,
+ 1.511464666000002,
+ 1.5110016249999987,
+ 1.51131775,
+ 1.511445166999998,
+ 1.5126569580000009,
+ 1.5140159170000018,
+ 1.6104141659999982,
+ 1.6125100829999965,
+ 1.6111711249999985,
+ 1.612879999999997,
+ 1.4968940829999937,
+ 1.4961242500000012,
+ 1.4942949580000047,
+ 1.4961574999999954,
+ 1.4933542079999995,
+ 1.4932850000000002,
+ 1.495215666,
+ 1.4941746249999994,
+ 1.4944899589999991,
+ 1.4968713330000014,
+ 1.4982131249999995,
+ 1.4975980829999997,
+ 1.4968774170000003,
+ 1.496618583,
+ 1.4952612920000021,
+ 1.4951001660000003,
+ 1.4972887499999956,
+ 1.498597457999999,
+ 1.500068290999998,
+ 1.4998819999999995
+ ],
+ "trace_id": "285ae9502aea442f99fe2dea27e72719",
+ "resolved_at": 1795163214.1534898,
+ "trace": {
+ "batches": [
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "8233dde3da6d4f13ab70ce40e99fc803"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-api"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "VaC2lZS0Jvk=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163147116546319",
+ "endTimeUnixNano": "1795163148619210223",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "zUOzMbzkwSk=",
+ "parentSpanId": "VaC2lZS0Jvk=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163147116618248",
+ "endTimeUnixNano": "1795163148618934756",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "VxiULXWaVJc=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163147116896045",
+ "endTimeUnixNano": "1795163148621553865",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "/+uJIGxCNXA=",
+ "parentSpanId": "VxiULXWaVJc=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163147116920508",
+ "endTimeUnixNano": "1795163148621330741",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "TLPZKH3DSqs=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163148628117246",
+ "endTimeUnixNano": "1795163150132835625",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "m+DXABYxyNw=",
+ "parentSpanId": "TLPZKH3DSqs=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163148628161379",
+ "endTimeUnixNano": "1795163150132398699",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "G3XjWBCNerg=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163148628411592",
+ "endTimeUnixNano": "1795163150134092770",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "d7j3PMuT/aE=",
+ "parentSpanId": "G3XjWBCNerg=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163148628444723",
+ "endTimeUnixNano": "1795163150131851998",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "uV4yHdysDKQ=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163150142550622",
+ "endTimeUnixNano": "1795163151645721387",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "GBDmlm8bziU=",
+ "parentSpanId": "uV4yHdysDKQ=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163150142615928",
+ "endTimeUnixNano": "1795163151645045841",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "/Q6sXHdTcTE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163150142938164",
+ "endTimeUnixNano": "1795163151647007530",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "jaTeKj+sb1o=",
+ "parentSpanId": "/Q6sXHdTcTE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163150142998343",
+ "endTimeUnixNano": "1795163151646582150",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "KZJVoAIlGGU=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163151653128255",
+ "endTimeUnixNano": "1795163153154988969",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "84Nns23Ooec=",
+ "parentSpanId": "KZJVoAIlGGU=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163151653189744",
+ "endTimeUnixNano": "1795163153154749353",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "eX0QwcdYFfE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163151653486976",
+ "endTimeUnixNano": "1795163153155467867",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "+rjZSFJPjzs=",
+ "parentSpanId": "eX0QwcdYFfE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163151653523911",
+ "endTimeUnixNano": "1795163153155262631",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "CAhIv+rchD8=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163153160722437",
+ "endTimeUnixNano": "1795163154663952272",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "lBd5omItFkw=",
+ "parentSpanId": "CAhIv+rchD8=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163153160766329",
+ "endTimeUnixNano": "1795163154663703857",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "uBBg5deuv4w=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163153161067975",
+ "endTimeUnixNano": "1795163154663117119",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ltBl+PMf5nQ=",
+ "parentSpanId": "uBBg5deuv4w=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163153161102651",
+ "endTimeUnixNano": "1795163154662849957",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "0RfCGv+5opQ=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163154668711205",
+ "endTimeUnixNano": "1795163156172276912",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "5R+lhVoUp80=",
+ "parentSpanId": "0RfCGv+5opQ=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163154668784006",
+ "endTimeUnixNano": "1795163156171530502",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Obv9Lbcf+QQ=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163154669339010",
+ "endTimeUnixNano": "1795163156171999316",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "2gaVEc6zXlM=",
+ "parentSpanId": "Obv9Lbcf+QQ=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163154669377680",
+ "endTimeUnixNano": "1795163156171106241",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "K7T8XtqnpEc=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163156178533132",
+ "endTimeUnixNano": "1795163157679739803",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "h1wRzuLcC6M=",
+ "parentSpanId": "K7T8XtqnpEc=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163156178600507",
+ "endTimeUnixNano": "1795163157679589733",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "U/Swk/plQJM=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163156179402924",
+ "endTimeUnixNano": "1795163157680688638",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ByzrCGZ3xno=",
+ "parentSpanId": "U/Swk/plQJM=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163156179465768",
+ "endTimeUnixNano": "1795163157680526664",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "QjpLfF9h9RE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163157682826505",
+ "endTimeUnixNano": "1795163159185277429",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "0vt9Bfevdb0=",
+ "parentSpanId": "QjpLfF9h9RE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163157682857777",
+ "endTimeUnixNano": "1795163159184788868",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "7X6+kN9Hcyg=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163157683025536",
+ "endTimeUnixNano": "1795163159185080874",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "z6V1ZLHKWHU=",
+ "parentSpanId": "7X6+kN9Hcyg=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163157683049262",
+ "endTimeUnixNano": "1795163159184465756",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "gIVhvsjLejQ=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163159190320535",
+ "endTimeUnixNano": "1795163160692623117",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "utQQuKrmAkM=",
+ "parentSpanId": "gIVhvsjLejQ=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163159190379564",
+ "endTimeUnixNano": "1795163160692297666",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "oadQZHFBDEM=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163159190708704",
+ "endTimeUnixNano": "1795163160693097779",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "FBScL6hLNhA=",
+ "parentSpanId": "oadQZHFBDEM=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163159190739266",
+ "endTimeUnixNano": "1795163160692919823",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ffAvuytvwcI=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163160699063565",
+ "endTimeUnixNano": "1795163162200470938",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "541iDkbPMag=",
+ "parentSpanId": "ffAvuytvwcI=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163160699119969",
+ "endTimeUnixNano": "1795163162200276498",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "l1s2jr4xd6c=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163160699344538",
+ "endTimeUnixNano": "1795163162200862872",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "u61nBmDUuWA=",
+ "parentSpanId": "l1s2jr4xd6c=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163160699373116",
+ "endTimeUnixNano": "1795163162200594141",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Jw0r03caAqM=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163162204180989",
+ "endTimeUnixNano": "1795163163707287503",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "9T+DnzEEnlQ=",
+ "parentSpanId": "Jw0r03caAqM=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163162204213598",
+ "endTimeUnixNano": "1795163163706605760",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "okR0pK9XAic=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163162204389560",
+ "endTimeUnixNano": "1795163163707587439",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "zCXB6WmCdz8=",
+ "parentSpanId": "okR0pK9XAic=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163162204420079",
+ "endTimeUnixNano": "1795163163706936037",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "/uPydmBGI9A=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163163716355521",
+ "endTimeUnixNano": "1795163165218659555",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "uV5Xbqch0Xw=",
+ "parentSpanId": "/uPydmBGI9A=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163163716417121",
+ "endTimeUnixNano": "1795163165218011289",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "EZV60sQ25NE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163163716718980",
+ "endTimeUnixNano": "1795163165218493531",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "oIhvADa82+g=",
+ "parentSpanId": "EZV60sQ25NE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163163716749154",
+ "endTimeUnixNano": "1795163165218296223",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "tazSwamC+2Q=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163165223303130",
+ "endTimeUnixNano": "1795163166724879672",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "tpwcedn/5es=",
+ "parentSpanId": "tazSwamC+2Q=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163165223349700",
+ "endTimeUnixNano": "1795163166724633221",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "y+AXzhlx2TY=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163165223584932",
+ "endTimeUnixNano": "1795163166725147334",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "RgDvDOe70aw=",
+ "parentSpanId": "y+AXzhlx2TY=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163165223615171",
+ "endTimeUnixNano": "1795163166725001101",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "/B3RW5F+4AE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163166730454786",
+ "endTimeUnixNano": "1795163168232138885",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "1uI2z9HnQoc=",
+ "parentSpanId": "/B3RW5F+4AE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163166730497587",
+ "endTimeUnixNano": "1795163168231806253",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "7vllanbVt5A=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163166730680984",
+ "endTimeUnixNano": "1795163168232235515",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "L423MDX/F/Q=",
+ "parentSpanId": "7vllanbVt5A=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163166730706998",
+ "endTimeUnixNano": "1795163168231966830",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "L/7z6lnUTA4=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163168236537361",
+ "endTimeUnixNano": "1795163169739248849",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "whd6oosb16s=",
+ "parentSpanId": "L/7z6lnUTA4=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163168236568751",
+ "endTimeUnixNano": "1795163169739076988",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "zwHGq8ezcSk=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163168236749071",
+ "endTimeUnixNano": "1795163169738797369",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "4R0KZmwEHGs=",
+ "parentSpanId": "zwHGq8ezcSk=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163168236782088",
+ "endTimeUnixNano": "1795163169738389711",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "cAScj+5RgkA=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163169743482650",
+ "endTimeUnixNano": "1795163171246023020",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Bk9wfFOc1/w=",
+ "parentSpanId": "cAScj+5RgkA=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163169743544125",
+ "endTimeUnixNano": "1795163171245745436",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "5EUy7II0gEg=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163169743943520",
+ "endTimeUnixNano": "1795163171245570115",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "dZfKIhcXc70=",
+ "parentSpanId": "5EUy7II0gEg=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163169743975989",
+ "endTimeUnixNano": "1795163171245351903",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "11a51b423246479e8ca2b43c8ec27a38"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-api"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "BDRJrbE6SA0=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163147116751354",
+ "endTimeUnixNano": "1795163148619812210",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Zf0ZGgLnqEM=",
+ "parentSpanId": "BDRJrbE6SA0=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163147116793986",
+ "endTimeUnixNano": "1795163148619561080",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "75P7jUjiO+E=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163147117082827",
+ "endTimeUnixNano": "1795163148619250772",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "GM0H1VSz+GM=",
+ "parentSpanId": "75P7jUjiO+E=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163147117108623",
+ "endTimeUnixNano": "1795163148618934547",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "zgn0I4R4isY=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163148627842570",
+ "endTimeUnixNano": "1795163150132489343",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "knON0o968Bc=",
+ "parentSpanId": "zgn0I4R4isY=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163148627903540",
+ "endTimeUnixNano": "1795163150131960605",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "6ERFwDPLSrA=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163148628250062",
+ "endTimeUnixNano": "1795163150133759407",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Vuw8sbYo5mM=",
+ "parentSpanId": "6ERFwDPLSrA=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163148628282777",
+ "endTimeUnixNano": "1795163150133344526",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ytVakbTdNEc=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163150142497694",
+ "endTimeUnixNano": "1795163151645583985",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "JyyaR/QX3jY=",
+ "parentSpanId": "ytVakbTdNEc=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163150142574586",
+ "endTimeUnixNano": "1795163151645053887",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "QSPpTwI14Rg=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163150143140832",
+ "endTimeUnixNano": "1795163151647004570",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "s7BknUiRfhU=",
+ "parentSpanId": "QSPpTwI14Rg=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163150143190926",
+ "endTimeUnixNano": "1795163151646570978",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "USz+ZFsjZao=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163151653066891",
+ "endTimeUnixNano": "1795163153154940491",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ZPmJk9O47v8=",
+ "parentSpanId": "USz+ZFsjZao=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163151653132007",
+ "endTimeUnixNano": "1795163153154543486",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "p63FdFi9qnw=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163151653634175",
+ "endTimeUnixNano": "1795163153155243652",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ELRTN99lsB0=",
+ "parentSpanId": "p63FdFi9qnw=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163151653673236",
+ "endTimeUnixNano": "1795163153155073976",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "RA6J/jK3DHE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163153160525954",
+ "endTimeUnixNano": "1795163154663138934",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "rdmV3g7Yw8A=",
+ "parentSpanId": "RA6J/jK3DHE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163153160578220",
+ "endTimeUnixNano": "1795163154662886862",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "m+ggRsTt1PY=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163153161023241",
+ "endTimeUnixNano": "1795163154663586753",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "OBOEz6f7XXY=",
+ "parentSpanId": "m+ggRsTt1PY=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163153161056570",
+ "endTimeUnixNano": "1795163154663381086",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "49kd22Nd3BU=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163154669275666",
+ "endTimeUnixNano": "1795163156172159279",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Y02pof0crRs=",
+ "parentSpanId": "49kd22Nd3BU=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163154669327409",
+ "endTimeUnixNano": "1795163156171741688",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "1C2mwUzqARY=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163154669596000",
+ "endTimeUnixNano": "1795163156172737336",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "1HDaup7+OA4=",
+ "parentSpanId": "1C2mwUzqARY=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163154669631770",
+ "endTimeUnixNano": "1795163156171316631",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ZSvo1EzKFlw=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163156177934309",
+ "endTimeUnixNano": "1795163157679858559",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "jGflidvTwfQ=",
+ "parentSpanId": "ZSvo1EzKFlw=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163156178007767",
+ "endTimeUnixNano": "1795163157679646869",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "BAG5TggdLOQ=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163156178566862",
+ "endTimeUnixNano": "1795163157680101019",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "F7FnsNWhlRY=",
+ "parentSpanId": "BAG5TggdLOQ=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163156178620309",
+ "endTimeUnixNano": "1795163157679467707",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Yt7/hO+J66U=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163157682822103",
+ "endTimeUnixNano": "1795163159185237490",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "N36mce4Wan0=",
+ "parentSpanId": "Yt7/hO+J66U=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163157682856142",
+ "endTimeUnixNano": "1795163159184475134",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "+si6mr+ngzE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163157683033123",
+ "endTimeUnixNano": "1795163159185104653",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "rQ4DjzObh9Y=",
+ "parentSpanId": "+si6mr+ngzE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163157683056640",
+ "endTimeUnixNano": "1795163159184913750",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "bKdF98QCkGU=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163159190521988",
+ "endTimeUnixNano": "1795163160692551441",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "TG/TJtrvwOk=",
+ "parentSpanId": "bKdF98QCkGU=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163159190579970",
+ "endTimeUnixNano": "1795163160692259882",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "dcls1LanVus=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163159190878298",
+ "endTimeUnixNano": "1795163160693748891",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "C5cKSjMfAR4=",
+ "parentSpanId": "dcls1LanVus=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163159190907394",
+ "endTimeUnixNano": "1795163160693418879",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "3R0W8dtF24g=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163160699036744",
+ "endTimeUnixNano": "1795163162200836325",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ziZ4du1Hozc=",
+ "parentSpanId": "3R0W8dtF24g=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163160699106495",
+ "endTimeUnixNano": "1795163162200281096",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "3pwZ/Phoso4=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163160699331357",
+ "endTimeUnixNano": "1795163162200646148",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "n7WYa2uioB4=",
+ "parentSpanId": "3pwZ/Phoso4=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163160699359434",
+ "endTimeUnixNano": "1795163162200457519",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "K9UmWB6QUw0=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163162204080278",
+ "endTimeUnixNano": "1795163163707753644",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "m1Ub8qmcn9Y=",
+ "parentSpanId": "K9UmWB6QUw0=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163162204115479",
+ "endTimeUnixNano": "1795163163707466119",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "k+HagEPAMik=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163162204600514",
+ "endTimeUnixNano": "1795163163706938879",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "MbiVLAoTnvg=",
+ "parentSpanId": "k+HagEPAMik=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163162204623800",
+ "endTimeUnixNano": "1795163163706594602",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "NKxiZYWty2w=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163163716382225",
+ "endTimeUnixNano": "1795163165218653457",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "a5VxaWzynsA=",
+ "parentSpanId": "NKxiZYWty2w=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163163716436763",
+ "endTimeUnixNano": "1795163165218015632",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "MQOoIopv91o=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163163716710538",
+ "endTimeUnixNano": "1795163165218466633",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "WP8lJKUvq48=",
+ "parentSpanId": "MQOoIopv91o=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163163716746813",
+ "endTimeUnixNano": "1795163165218263144",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "y4cHvrN34BE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163165223106783",
+ "endTimeUnixNano": "1795163166725023274",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "2qIAxqcyejU=",
+ "parentSpanId": "y4cHvrN34BE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163165223161038",
+ "endTimeUnixNano": "1795163166724932369",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "VBjvchwTDA0=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163165223446390",
+ "endTimeUnixNano": "1795163166724831275",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "7hsE/emI1oc=",
+ "parentSpanId": "VBjvchwTDA0=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163165223471325",
+ "endTimeUnixNano": "1795163166724635017",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "MkEaC8VBx1U=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163166730450151",
+ "endTimeUnixNano": "1795163168232101193",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "szCJHhhPuhs=",
+ "parentSpanId": "MkEaC8VBx1U=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163166730498589",
+ "endTimeUnixNano": "1795163168231962113",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "TdA4Lsrqgf8=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163166730704785",
+ "endTimeUnixNano": "1795163168232203291",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "2GROAXT4irw=",
+ "parentSpanId": "TdA4Lsrqgf8=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163166730728629",
+ "endTimeUnixNano": "1795163168231791268",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Ch3eXSTSzTc=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163168236517827",
+ "endTimeUnixNano": "1795163169739179863",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "QpflcQreyrM=",
+ "parentSpanId": "Ch3eXSTSzTc=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163168236552973",
+ "endTimeUnixNano": "1795163169738989180",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ht56Tnnr8wE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163168236737300",
+ "endTimeUnixNano": "1795163169738833010",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "cwpCci+9+F0=",
+ "parentSpanId": "ht56Tnnr8wE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163168236763346",
+ "endTimeUnixNano": "1795163169738461744",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "3AqsChofLYI=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163169743485697",
+ "endTimeUnixNano": "1795163171245995399",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ig/ZF+T7RHw=",
+ "parentSpanId": "3AqsChofLYI=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163169743543666",
+ "endTimeUnixNano": "1795163171245840606",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "6VQ3SkD8rjY=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163169743971649",
+ "endTimeUnixNano": "1795163171245664409",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "8vvjRAIAA/Q=",
+ "parentSpanId": "6VQ3SkD8rjY=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163169744003116",
+ "endTimeUnixNano": "1795163171245343266",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "8233dde3da6d4f13ab70ce40e99fc803"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-api"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "xVSbaWHLtgs=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163171251241799",
+ "endTimeUnixNano": "1795163172753298069",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "9lvt9C2OVa8=",
+ "parentSpanId": "xVSbaWHLtgs=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163171251289155",
+ "endTimeUnixNano": "1795163172752576001",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "3FVikXMEig0=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163171251628615",
+ "endTimeUnixNano": "1795163172753054307",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "0wQRQmvaJsU=",
+ "parentSpanId": "3FVikXMEig0=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163171251649560",
+ "endTimeUnixNano": "1795163172752885888",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "9cjvGElVZmM=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163172757701857",
+ "endTimeUnixNano": "1795163174259607406",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "L4eG8aeQJTI=",
+ "parentSpanId": "9cjvGElVZmM=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163172757738945",
+ "endTimeUnixNano": "1795163174258882420",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "+ey/l71B8Bk=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163172757880788",
+ "endTimeUnixNano": "1795163174259439977",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Na2Wnq8up4c=",
+ "parentSpanId": "+ey/l71B8Bk=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163172757896892",
+ "endTimeUnixNano": "1795163174259173441",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "tiySnmwn6JE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163174264893827",
+ "endTimeUnixNano": "1795163175768909530",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "XQoFGd02FiE=",
+ "parentSpanId": "tiySnmwn6JE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163174264963444",
+ "endTimeUnixNano": "1795163175768519140",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "mO2gcZ5yR5Q=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163174265288584",
+ "endTimeUnixNano": "1795163175768073277",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "pUWy9CPFoAk=",
+ "parentSpanId": "mO2gcZ5yR5Q=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163174265316072",
+ "endTimeUnixNano": "1795163175767268888",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "uJNFgzM7dy4=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163175774989451",
+ "endTimeUnixNano": "1795163177277887215",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "W5ssEjJ3cxE=",
+ "parentSpanId": "uJNFgzM7dy4=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163175775065902",
+ "endTimeUnixNano": "1795163177277496036",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "jlbOiTYw2wI=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163175775340927",
+ "endTimeUnixNano": "1795163177277206030",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ycveYyipuLg=",
+ "parentSpanId": "jlbOiTYw2wI=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163175775378173",
+ "endTimeUnixNano": "1795163177276792540",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "O1/JPc+SrbA=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163177284299715",
+ "endTimeUnixNano": "1795163178787210196",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "25VANoU3fpE=",
+ "parentSpanId": "O1/JPc+SrbA=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163177284351428",
+ "endTimeUnixNano": "1795163178786171870",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "tyIYqKMoFFg=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163177284939197",
+ "endTimeUnixNano": "1795163178786965575",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "nuBRzH39M2k=",
+ "parentSpanId": "tyIYqKMoFFg=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163177284977397",
+ "endTimeUnixNano": "1795163178786584006",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "bzDfwmRJ+0k=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163178794085901",
+ "endTimeUnixNano": "1795163180297653821",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Z4w4EtjZis0=",
+ "parentSpanId": "bzDfwmRJ+0k=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163178794167177",
+ "endTimeUnixNano": "1795163180297360557",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "HIJS4+1e+yo=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163178794624851",
+ "endTimeUnixNano": "1795163180296623499",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "0u/NFNRwzO0=",
+ "parentSpanId": "HIJS4+1e+yo=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163178794668513",
+ "endTimeUnixNano": "1795163180296152143",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "6IFhWj2pwbY=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163180305097998",
+ "endTimeUnixNano": "1795163181294372214",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ZM6pRnincpo=",
+ "parentSpanId": "6IFhWj2pwbY=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163180305164548",
+ "endTimeUnixNano": "1795163181294033719",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "2qZZsbaV3Gw=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163180305454726",
+ "endTimeUnixNano": "1795163181293877564",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "WiXkJHtLaxA=",
+ "parentSpanId": "2qZZsbaV3Gw=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163180305497800",
+ "endTimeUnixNano": "1795163181293346012",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "U2B6DzKKC94=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181297328565",
+ "endTimeUnixNano": "1795163181298991687",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "wDkegWjbgxg=",
+ "parentSpanId": "U2B6DzKKC94=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181297372013",
+ "endTimeUnixNano": "1795163181298641225",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ATOyh+llU4A=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181298285759",
+ "endTimeUnixNano": "1795163181299375465",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "alHlYpamIyI=",
+ "parentSpanId": "ATOyh+llU4A=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181298316865",
+ "endTimeUnixNano": "1795163181299216100",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "D4koG1Vqk/Q=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181301127444",
+ "endTimeUnixNano": "1795163181302284032",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "eZahb5gVFvw=",
+ "parentSpanId": "D4koG1Vqk/Q=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181301163303",
+ "endTimeUnixNano": "1795163181301977393",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "0mFtpvbKjfo=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181301322335",
+ "endTimeUnixNano": "1795163181302130546",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "OqPg+jEBfCE=",
+ "parentSpanId": "0mFtpvbKjfo=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181301344851",
+ "endTimeUnixNano": "1795163181301886285",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "9fqZEe+0Fzk=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181304861734",
+ "endTimeUnixNano": "1795163181306034834",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "B8/6zJoAvTo=",
+ "parentSpanId": "9fqZEe+0Fzk=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181304893090",
+ "endTimeUnixNano": "1795163181305939473",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ArY2pFy8t0o=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181305067925",
+ "endTimeUnixNano": "1795163181305894107",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "i8HQSHdjMps=",
+ "parentSpanId": "ArY2pFy8t0o=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181305090024",
+ "endTimeUnixNano": "1795163181305634836",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "iO+BfylFenY=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181308352639",
+ "endTimeUnixNano": "1795163181309185785",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "b88r/EHiB1w=",
+ "parentSpanId": "iO+BfylFenY=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181308384412",
+ "endTimeUnixNano": "1795163181309016287",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "TThB/0i247Q=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181308590270",
+ "endTimeUnixNano": "1795163181309389432",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "vMwXFjKOVFs=",
+ "parentSpanId": "TThB/0i247Q=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181308610159",
+ "endTimeUnixNano": "1795163181309281437",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "75dfyEEPxks=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181311871231",
+ "endTimeUnixNano": "1795163181313064888",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "T0/ovocfQpQ=",
+ "parentSpanId": "75dfyEEPxks=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181311910718",
+ "endTimeUnixNano": "1795163181312933835",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "P1j8GCOBF0U=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181312120203",
+ "endTimeUnixNano": "1795163181312858280",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "0OPBCmKqtoM=",
+ "parentSpanId": "P1j8GCOBF0U=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181312143262",
+ "endTimeUnixNano": "1795163181312647252",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "2VHF9u711+Y=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181315995179",
+ "endTimeUnixNano": "1795163181317215688",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "QNEke99SvRU=",
+ "parentSpanId": "2VHF9u711+Y=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181316029412",
+ "endTimeUnixNano": "1795163181317109778",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "9Y2Z14h4sow=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181316179646",
+ "endTimeUnixNano": "1795163181317042688",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "iIF4XKnplhA=",
+ "parentSpanId": "9Y2Z14h4sow=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181316195741",
+ "endTimeUnixNano": "1795163181316831326",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "DjY7D2sWZqA=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181319442010",
+ "endTimeUnixNano": "1795163181321084993",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "DwPCNOr0NG0=",
+ "parentSpanId": "DjY7D2sWZqA=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181319468780",
+ "endTimeUnixNano": "1795163181320948477",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "1fqw8UyCUnA=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181319833002",
+ "endTimeUnixNano": "1795163181320872380",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "28DJaDb3EfU=",
+ "parentSpanId": "1fqw8UyCUnA=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181319852725",
+ "endTimeUnixNano": "1795163181320547853",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ivNXyxhOHHY=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181323409970",
+ "endTimeUnixNano": "1795163181324448556",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "QicN5PRLXBQ=",
+ "parentSpanId": "ivNXyxhOHHY=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181323436989",
+ "endTimeUnixNano": "1795163181324357573",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "I+8egs3Sw78=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181323592769",
+ "endTimeUnixNano": "1795163181324280934",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "CEKW+Jhb0Vs=",
+ "parentSpanId": "I+8egs3Sw78=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181323613200",
+ "endTimeUnixNano": "1795163181324106516",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "KXbie6whVy4=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181326592818",
+ "endTimeUnixNano": "1795163181327744653",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "bXFh0zdHDnI=",
+ "parentSpanId": "KXbie6whVy4=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181326619004",
+ "endTimeUnixNano": "1795163181327660968",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "+VweUCXyLzE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181326835619",
+ "endTimeUnixNano": "1795163181327613141",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "9qKh12703AE=",
+ "parentSpanId": "+VweUCXyLzE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181326854758",
+ "endTimeUnixNano": "1795163181327406992",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "11a51b423246479e8ca2b43c8ec27a38"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-api"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "L7mrMbjWV08=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163171251513333",
+ "endTimeUnixNano": "1795163172753451928",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "A8qJZLpasrI=",
+ "parentSpanId": "L7mrMbjWV08=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163171251553930",
+ "endTimeUnixNano": "1795163172752744920",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "qKNMOg5kd+8=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163171251791961",
+ "endTimeUnixNano": "1795163172753224561",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "MVaS7EBy3Ac=",
+ "parentSpanId": "qKNMOg5kd+8=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163171251817579",
+ "endTimeUnixNano": "1795163172752577420",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "RZ3mfP06Y5U=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163172757318879",
+ "endTimeUnixNano": "1795163174259415367",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "SkTiSLGumWQ=",
+ "parentSpanId": "RZ3mfP06Y5U=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163172757359680",
+ "endTimeUnixNano": "1795163174258886675",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "/8xDew9Vpx8=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163172757603692",
+ "endTimeUnixNano": "1795163174259721988",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "JgKYI64vbPY=",
+ "parentSpanId": "/8xDew9Vpx8=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163172757626721",
+ "endTimeUnixNano": "1795163174259154212",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "SW/GOektqFs=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163174265096044",
+ "endTimeUnixNano": "1795163175768890970",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "VGPUS1x3g8s=",
+ "parentSpanId": "SW/GOektqFs=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163174265155358",
+ "endTimeUnixNano": "1795163175767256125",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "87LWzY58A24=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163174265424521",
+ "endTimeUnixNano": "1795163175769248453",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "xwQDy//+TdA=",
+ "parentSpanId": "87LWzY58A24=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163174265449632",
+ "endTimeUnixNano": "1795163175768260798",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "cin2LeQ9t/Q=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163175775019147",
+ "endTimeUnixNano": "1795163177277130797",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "tNd7iff40Co=",
+ "parentSpanId": "cin2LeQ9t/Q=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163175775084421",
+ "endTimeUnixNano": "1795163177276844878",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "DMnrK3fBh/c=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163175775347225",
+ "endTimeUnixNano": "1795163177277724196",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "i30xWvk797w=",
+ "parentSpanId": "DMnrK3fBh/c=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163175775383220",
+ "endTimeUnixNano": "1795163177277393988",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "aOJANJqIvYI=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163177284057543",
+ "endTimeUnixNano": "1795163178787509697",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "+LYbAN3MqfE=",
+ "parentSpanId": "aOJANJqIvYI=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163177284114552",
+ "endTimeUnixNano": "1795163178786513030",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "z67d+K2I468=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163177284437796",
+ "endTimeUnixNano": "1795163178786967702",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "1ubq9gKFdgE=",
+ "parentSpanId": "z67d+K2I468=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163177284464236",
+ "endTimeUnixNano": "1795163178786099727",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "R8pPe9/QjXw=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163178794386902",
+ "endTimeUnixNano": "1795163180296690508",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ntd8tkHvq+o=",
+ "parentSpanId": "R8pPe9/QjXw=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163178794458545",
+ "endTimeUnixNano": "1795163180296201430",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "lDKbKydQIj8=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163178794804376",
+ "endTimeUnixNano": "1795163180297720455",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "503"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "GB15iM+Uj+0=",
+ "parentSpanId": "lDKbKydQIj8=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163178794851374",
+ "endTimeUnixNano": "1795163180297439617",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "LockNotAvailable"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "M4UYlvZLxj8=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163180305122725",
+ "endTimeUnixNano": "1795163181294059487",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "57S7wnM3HC0=",
+ "parentSpanId": "M4UYlvZLxj8=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163180305181728",
+ "endTimeUnixNano": "1795163181293842914",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "muByvK56/kQ=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163180305464358",
+ "endTimeUnixNano": "1795163181293227342",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "frBgqp33XsI=",
+ "parentSpanId": "muByvK56/kQ=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163180305507808",
+ "endTimeUnixNano": "1795163181292810874",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Yn5/F2QPMIw=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181297216776",
+ "endTimeUnixNano": "1795163181298595609",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "B6QRDa/Glw4=",
+ "parentSpanId": "Yn5/F2QPMIw=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181297266187",
+ "endTimeUnixNano": "1795163181298111758",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "2vJ1vQSVzvY=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181297535548",
+ "endTimeUnixNano": "1795163181298447626",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "dGtyCbA8d1o=",
+ "parentSpanId": "2vJ1vQSVzvY=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181297562443",
+ "endTimeUnixNano": "1795163181298196778",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "R8bK9JbeBqY=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181301022534",
+ "endTimeUnixNano": "1795163181302309300",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "cGE4aaQh/z4=",
+ "parentSpanId": "R8bK9JbeBqY=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181301058435",
+ "endTimeUnixNano": "1795163181301969929",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "18f0Hz1jKpo=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181301291854",
+ "endTimeUnixNano": "1795163181302140011",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "j5BnlpBFnzs=",
+ "parentSpanId": "18f0Hz1jKpo=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181301313120",
+ "endTimeUnixNano": "1795163181301813149",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "LwmQdMzx/Kk=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181304825457",
+ "endTimeUnixNano": "1795163181305812381",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "EqtPPKVQfN4=",
+ "parentSpanId": "LwmQdMzx/Kk=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181304863068",
+ "endTimeUnixNano": "1795163181305624328",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "bOpJr4GcUxI=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181305063797",
+ "endTimeUnixNano": "1795163181306043173",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ROQ91j6NZZ8=",
+ "parentSpanId": "bOpJr4GcUxI=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181305085187",
+ "endTimeUnixNano": "1795163181305896901",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "mqmcCUTJOUw=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181308344008",
+ "endTimeUnixNano": "1795163181309371503",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "mTRYryPInIE=",
+ "parentSpanId": "mqmcCUTJOUw=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181308374572",
+ "endTimeUnixNano": "1795163181309206299",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ORmxOdIWIOw=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181308511671",
+ "endTimeUnixNano": "1795163181309481082",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "a61aM67LFI4=",
+ "parentSpanId": "ORmxOdIWIOw=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181308526807",
+ "endTimeUnixNano": "1795163181309056691",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "553mCS6Xm7E=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181311822696",
+ "endTimeUnixNano": "1795163181312913236",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "Y2pCKV9Qle4=",
+ "parentSpanId": "553mCS6Xm7E=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181311855470",
+ "endTimeUnixNano": "1795163181312694161",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "nK1FdeL1MVE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181312160733",
+ "endTimeUnixNano": "1795163181313119219",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "rxCMd+B+J1M=",
+ "parentSpanId": "nK1FdeL1MVE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181312188628",
+ "endTimeUnixNano": "1795163181312983120",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "qFE9uvWQPEU=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181315969994",
+ "endTimeUnixNano": "1795163181317012207",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "mMUiRsJF5yY=",
+ "parentSpanId": "qFE9uvWQPEU=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181316009564",
+ "endTimeUnixNano": "1795163181316728168",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "pBpOJlA1H0A=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181316316203",
+ "endTimeUnixNano": "1795163181317220942",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "FVj8LCcbOJs=",
+ "parentSpanId": "pBpOJlA1H0A=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181316337343",
+ "endTimeUnixNano": "1795163181317092182",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "t+cVr1L0dkc=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181319452726",
+ "endTimeUnixNano": "1795163181320550230",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "EWYfyoCBMnU=",
+ "parentSpanId": "t+cVr1L0dkc=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181319477703",
+ "endTimeUnixNano": "1795163181320363512",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "HyisMMqdpUQ=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181319845011",
+ "endTimeUnixNano": "1795163181320981042",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "ocKHQL/k1as=",
+ "parentSpanId": "HyisMMqdpUQ=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181319899050",
+ "endTimeUnixNano": "1795163181320741035",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "GcD1VDquVvo=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181323139690",
+ "endTimeUnixNano": "1795163181324335932",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "jyQ5DhaeUf8=",
+ "parentSpanId": "GcD1VDquVvo=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181323178969",
+ "endTimeUnixNano": "1795163181324184239",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "f52RA6I5rME=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181323627961",
+ "endTimeUnixNano": "1795163181324425414",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "JPZBomgwSS0=",
+ "parentSpanId": "f52RA6I5rME=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181323647600",
+ "endTimeUnixNano": "1795163181324102972",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "4VyjKvR+MiE=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181326554874",
+ "endTimeUnixNano": "1795163181327679731",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "vOdwDxuOido=",
+ "parentSpanId": "4VyjKvR+MiE=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181326583186",
+ "endTimeUnixNano": "1795163181327532833",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "P3UUZEmed+w=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "GET /api/v1/users/me",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795163181326997111",
+ "endTimeUnixNano": "1795163181327813495",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "GET"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/users/me"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "200"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==",
+ "spanId": "W4CaB5NCoaE=",
+ "parentSpanId": "P3UUZEmed+w=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795163181327018460",
+ "endTimeUnixNano": "1795163181327446229",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "diagnosis": "Server 5xx spans contain slow/error PostgreSQL transaction; health remains available; sessions lock is the bottleneck."
+ }
+}
diff --git a/evidence/observability-cost.json b/evidence/observability-cost.json
new file mode 100644
index 0000000..f5ce401
--- /dev/null
+++ b/evidence/observability-cost.json
@@ -0,0 +1,1692 @@
+{
+ "method": "Fixed arrival mixed requests, 5% image uploads, 15% messages, 80% profile reads; same fixture/resources. No transparent retries.",
+ "notes": [
+ "CPU cgroup total across load plus exporter drain; includes renderer.",
+ "Anonymous/working-set memory sampled at 1Hz through read-only Docker stats API, not exact per-PID peak RSS.",
+ "Collector rx bytes include OTLP HTTP, Prometheus scrapes and backend replies; accepted signal counts are exact Collector counter deltas.",
+ "Fresh API/worker/dispatcher containers per setting; data/backends retained. Per-instance counter deltas ignore expired old series.",
+ "Initial post-window SQL-lock probes are classified by their actual HTTP status; Docker exec startup consumed part of the hold time. The separate sampling-retention.json study synchronizes on pg_locks and guarantees 3x503 + 1xslow200 per repeat."
+ ],
+ "runs": [
+ {
+ "mode": "off",
+ "enabled": 0,
+ "sample_ratio": 0,
+ "repeat": 0,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.975265541,
+ "successful_rps": 30.037147629826347,
+ "p50_seconds": 0.00300129199999688,
+ "p95_seconds": 0.008791667000000558,
+ "p99_seconds": 0.014328458000001376,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.397554,
+ "peak_sampled_anonymous_bytes": 178855936,
+ "peak_sampled_working_set_bytes": 187908096,
+ "rx_bytes": 4530061
+ },
+ "broker": {
+ "cpu_seconds": 0.15378800000000004,
+ "peak_sampled_anonymous_bytes": 4116480,
+ "peak_sampled_working_set_bytes": 9412608,
+ "rx_bytes": 43590
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.6928389999999999,
+ "peak_sampled_anonymous_bytes": 44990464,
+ "peak_sampled_working_set_bytes": 47325184,
+ "rx_bytes": 458002
+ },
+ "grafana": {
+ "cpu_seconds": 0.04605400000000026,
+ "peak_sampled_anonymous_bytes": 83767296,
+ "peak_sampled_working_set_bytes": 89759744,
+ "rx_bytes": 84
+ },
+ "lb": {
+ "cpu_seconds": 0.24711599999999967,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18386944,
+ "rx_bytes": 4325424
+ },
+ "loki": {
+ "cpu_seconds": 0.1151819999999999,
+ "peak_sampled_anonymous_bytes": 62185472,
+ "peak_sampled_working_set_bytes": 63283200,
+ "rx_bytes": 370
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.04942599999999997,
+ "peak_sampled_anonymous_bytes": 58355712,
+ "peak_sampled_working_set_bytes": 63848448,
+ "rx_bytes": 14437
+ },
+ "postgres": {
+ "cpu_seconds": 1.5883010000000013,
+ "peak_sampled_anonymous_bytes": 15314944,
+ "peak_sampled_working_set_bytes": 88096768,
+ "rx_bytes": 1178507
+ },
+ "prometheus": {
+ "cpu_seconds": 0.025018000000000096,
+ "peak_sampled_anonymous_bytes": 30494720,
+ "peak_sampled_working_set_bytes": 43393024,
+ "rx_bytes": 64749
+ },
+ "redis": {
+ "cpu_seconds": 0.16833299999999962,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14114816,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.44473300000000027,
+ "peak_sampled_anonymous_bytes": 69632000,
+ "peak_sampled_working_set_bytes": 75223040,
+ "rx_bytes": 11145201
+ },
+ "s3-store": {
+ "cpu_seconds": 0.4075490000000004,
+ "peak_sampled_anonymous_bytes": 137076736,
+ "peak_sampled_working_set_bytes": 241938432,
+ "rx_bytes": 7487615
+ },
+ "tempo": {
+ "cpu_seconds": 0.061722,
+ "peak_sampled_anonymous_bytes": 270999552,
+ "peak_sampled_working_set_bytes": 272510976,
+ "rx_bytes": 506
+ },
+ "worker": {
+ "cpu_seconds": 5.2742759999999995,
+ "peak_sampled_anonymous_bytes": 80060416,
+ "peak_sampled_working_set_bytes": 84627456,
+ "rx_bytes": 3739196
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 0,
+ "otelcol_receiver_accepted_log_records_total": 0,
+ "otelcol_receiver_accepted_metric_points_total": 0,
+ "otelcol_exporter_sent_spans_total": 0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "5c230696e6c348e490b5870477deb25f",
+ "status": 200,
+ "latency_seconds": 0.00486374999999839,
+ "trace_sampled": false
+ },
+ {
+ "request_id": "0893cd3644994ecf91a14845d1034676",
+ "status": 200,
+ "latency_seconds": 0.003821957999996073,
+ "trace_sampled": false
+ },
+ {
+ "request_id": "73d9ece550724c04890a28ba8990b480",
+ "status": 200,
+ "latency_seconds": 0.005794915999999262,
+ "trace_sampled": false
+ },
+ {
+ "request_id": "db40195042b4493eb1e44d7dc20b96ba",
+ "status": 200,
+ "latency_seconds": 0.006047958999999992,
+ "trace_sampled": false
+ }
+ ]
+ },
+ {
+ "mode": "full",
+ "enabled": 1,
+ "sample_ratio": 1,
+ "repeat": 0,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.971939667,
+ "successful_rps": 30.042149636141296,
+ "p50_seconds": 0.002889416000002143,
+ "p95_seconds": 0.010209084000010193,
+ "p99_seconds": 0.028965249999998832,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 3.328923,
+ "peak_sampled_anonymous_bytes": 155086848,
+ "peak_sampled_working_set_bytes": 163901440,
+ "rx_bytes": 4581068
+ },
+ "broker": {
+ "cpu_seconds": 0.17730200000000007,
+ "peak_sampled_anonymous_bytes": 4132864,
+ "peak_sampled_working_set_bytes": 9302016,
+ "rx_bytes": 45306
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.8315530000000001,
+ "peak_sampled_anonymous_bytes": 45432832,
+ "peak_sampled_working_set_bytes": 47874048,
+ "rx_bytes": 482273
+ },
+ "grafana": {
+ "cpu_seconds": 0.06061899999999998,
+ "peak_sampled_anonymous_bytes": 83779584,
+ "peak_sampled_working_set_bytes": 89772032,
+ "rx_bytes": 84
+ },
+ "lb": {
+ "cpu_seconds": 0.2562519999999999,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18386944,
+ "rx_bytes": 4286332
+ },
+ "loki": {
+ "cpu_seconds": 0.18902600000000014,
+ "peak_sampled_anonymous_bytes": 65634304,
+ "peak_sampled_working_set_bytes": 66740224,
+ "rx_bytes": 108942
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.19410300000000014,
+ "peak_sampled_anonymous_bytes": 57430016,
+ "peak_sampled_working_set_bytes": 62943232,
+ "rx_bytes": 1092634
+ },
+ "postgres": {
+ "cpu_seconds": 1.645151999999996,
+ "peak_sampled_anonymous_bytes": 16187392,
+ "peak_sampled_working_set_bytes": 89083904,
+ "rx_bytes": 1197385
+ },
+ "prometheus": {
+ "cpu_seconds": 0.04396699999999987,
+ "peak_sampled_anonymous_bytes": 30601216,
+ "peak_sampled_working_set_bytes": 43503616,
+ "rx_bytes": 67364
+ },
+ "redis": {
+ "cpu_seconds": 0.21558100000000024,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14110720,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.4803259999999998,
+ "peak_sampled_anonymous_bytes": 70320128,
+ "peak_sampled_working_set_bytes": 75911168,
+ "rx_bytes": 11145738
+ },
+ "s3-store": {
+ "cpu_seconds": 0.473001,
+ "peak_sampled_anonymous_bytes": 146186240,
+ "peak_sampled_working_set_bytes": 251293696,
+ "rx_bytes": 7488458
+ },
+ "tempo": {
+ "cpu_seconds": 0.08789200000000003,
+ "peak_sampled_anonymous_bytes": 285114368,
+ "peak_sampled_working_set_bytes": 286642176,
+ "rx_bytes": 89778
+ },
+ "worker": {
+ "cpu_seconds": 5.533063,
+ "peak_sampled_anonymous_bytes": 77615104,
+ "peak_sampled_working_set_bytes": 82518016,
+ "rx_bytes": 3768611
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 1950.0,
+ "otelcol_receiver_accepted_log_records_total": 840.0,
+ "otelcol_receiver_accepted_metric_points_total": 1011.0,
+ "otelcol_exporter_sent_spans_total": 1950.0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 1950.0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "723eac4a47374627beac821a9091b1a8",
+ "status": 200,
+ "latency_seconds": 0.007174290999998334,
+ "trace_sampled": true,
+ "trace_id": "cffabf0fc53c5709f7d40b18204dfd3b"
+ },
+ {
+ "request_id": "57a2700ea7f649869da7efab6360ef81",
+ "status": 503,
+ "latency_seconds": 1.4987169580000028,
+ "trace_sampled": true,
+ "trace_id": "33caaa68b816a492a810f84630616a7a"
+ },
+ {
+ "request_id": "719f8df8ee3143be8b73676766611b58",
+ "status": 200,
+ "latency_seconds": 0.007091125000002307,
+ "trace_sampled": true,
+ "trace_id": "7dad2fe9e5ad5a1bbd567765ab64dd46"
+ },
+ {
+ "request_id": "f3b88188679d44f18893ab04e41a26b4",
+ "status": 200,
+ "latency_seconds": 0.005517166999993606,
+ "trace_sampled": true,
+ "trace_id": "4ea7ac61e1caaa90d2b31e514698a8da"
+ }
+ ]
+ },
+ {
+ "mode": "sample10",
+ "enabled": 1,
+ "sample_ratio": 0.1,
+ "repeat": 0,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.972699833000007,
+ "successful_rps": 30.041006224338613,
+ "p50_seconds": 0.0027112500000043838,
+ "p95_seconds": 0.00865920799999742,
+ "p99_seconds": 0.013731374999991885,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.523611,
+ "peak_sampled_anonymous_bytes": 176848896,
+ "peak_sampled_working_set_bytes": 186454016,
+ "rx_bytes": 4571688
+ },
+ "broker": {
+ "cpu_seconds": 0.15326200000000068,
+ "peak_sampled_anonymous_bytes": 3239936,
+ "peak_sampled_working_set_bytes": 8257536,
+ "rx_bytes": 45750
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.730053,
+ "peak_sampled_anonymous_bytes": 45420544,
+ "peak_sampled_working_set_bytes": 47857664,
+ "rx_bytes": 471320
+ },
+ "grafana": {
+ "cpu_seconds": 0.09900699999999985,
+ "peak_sampled_anonymous_bytes": 84144128,
+ "peak_sampled_working_set_bytes": 90238976,
+ "rx_bytes": 5856
+ },
+ "lb": {
+ "cpu_seconds": 0.23503000000000007,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18407424,
+ "rx_bytes": 4285664
+ },
+ "loki": {
+ "cpu_seconds": 0.1507130000000001,
+ "peak_sampled_anonymous_bytes": 72830976,
+ "peak_sampled_working_set_bytes": 73965568,
+ "rx_bytes": 110022
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.16146100000000008,
+ "peak_sampled_anonymous_bytes": 61222912,
+ "peak_sampled_working_set_bytes": 66953216,
+ "rx_bytes": 738170
+ },
+ "postgres": {
+ "cpu_seconds": 1.5299630000000022,
+ "peak_sampled_anonymous_bytes": 14110720,
+ "peak_sampled_working_set_bytes": 86786048,
+ "rx_bytes": 1168273
+ },
+ "prometheus": {
+ "cpu_seconds": 0.03131799999999996,
+ "peak_sampled_anonymous_bytes": 31059968,
+ "peak_sampled_working_set_bytes": 43962368,
+ "rx_bytes": 93821
+ },
+ "redis": {
+ "cpu_seconds": 0.1691539999999998,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14114816,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.4351940000000001,
+ "peak_sampled_anonymous_bytes": 70418432,
+ "peak_sampled_working_set_bytes": 76009472,
+ "rx_bytes": 11144860
+ },
+ "s3-store": {
+ "cpu_seconds": 0.39130299999999973,
+ "peak_sampled_anonymous_bytes": 147214336,
+ "peak_sampled_working_set_bytes": 252542976,
+ "rx_bytes": 7486024
+ },
+ "tempo": {
+ "cpu_seconds": 0.07223400000000002,
+ "peak_sampled_anonymous_bytes": 167501824,
+ "peak_sampled_working_set_bytes": 169185280,
+ "rx_bytes": 23025
+ },
+ "worker": {
+ "cpu_seconds": 5.2699810000000005,
+ "peak_sampled_anonymous_bytes": 76886016,
+ "peak_sampled_working_set_bytes": 81866752,
+ "rx_bytes": 3758994
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 176.0,
+ "otelcol_receiver_accepted_log_records_total": 840.0,
+ "otelcol_receiver_accepted_metric_points_total": 956.0,
+ "otelcol_exporter_sent_spans_total": 176.0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 176.0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "ffcece7a06fb4cd6b066e95e4103df70",
+ "status": 200,
+ "latency_seconds": 0.003515332999995735,
+ "trace_sampled": false,
+ "trace_id": "c4dc83fcda1c1ed05c81c8e1fc22d83a"
+ },
+ {
+ "request_id": "b36d8dd933364080a5fec3f8741e6fe5",
+ "status": 503,
+ "latency_seconds": 1.5039899999999875,
+ "trace_sampled": false,
+ "trace_id": "975ddde505a8354dc1754074bea788d4"
+ },
+ {
+ "request_id": "e879d9d8247b42c1bcf8ab6c87a75083",
+ "status": 200,
+ "latency_seconds": 0.00465941599998132,
+ "trace_sampled": false,
+ "trace_id": "8faa8d99efcfc13c1e0d1cb504b842e6"
+ },
+ {
+ "request_id": "89c845e71c7c4de28af6b9eaca054987",
+ "status": 200,
+ "latency_seconds": 0.6065338749999967,
+ "trace_sampled": false,
+ "trace_id": "5fda13d81e8a026e1b50f10a6064968c"
+ }
+ ]
+ },
+ {
+ "mode": "sample25",
+ "enabled": 1,
+ "sample_ratio": 0.25,
+ "repeat": 0,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.97326854100001,
+ "successful_rps": 30.040150853044082,
+ "p50_seconds": 0.0028638750000027358,
+ "p95_seconds": 0.008952082999996946,
+ "p99_seconds": 0.014915040999994744,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.5460709999999995,
+ "peak_sampled_anonymous_bytes": 167587840,
+ "peak_sampled_working_set_bytes": 176685056,
+ "rx_bytes": 4573821
+ },
+ "broker": {
+ "cpu_seconds": 0.16021000000000019,
+ "peak_sampled_anonymous_bytes": 3235840,
+ "peak_sampled_working_set_bytes": 8249344,
+ "rx_bytes": 45750
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.7868260000000001,
+ "peak_sampled_anonymous_bytes": 45428736,
+ "peak_sampled_working_set_bytes": 48144384,
+ "rx_bytes": 468546
+ },
+ "grafana": {
+ "cpu_seconds": 0.04848399999999975,
+ "peak_sampled_anonymous_bytes": 84168704,
+ "peak_sampled_working_set_bytes": 90267648,
+ "rx_bytes": 234
+ },
+ "lb": {
+ "cpu_seconds": 0.2286579999999998,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18644992,
+ "rx_bytes": 4284384
+ },
+ "loki": {
+ "cpu_seconds": 0.14923700000000029,
+ "peak_sampled_anonymous_bytes": 75874304,
+ "peak_sampled_working_set_bytes": 77029376,
+ "rx_bytes": 109617
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.17502400000000007,
+ "peak_sampled_anonymous_bytes": 61571072,
+ "peak_sampled_working_set_bytes": 67301376,
+ "rx_bytes": 804572
+ },
+ "postgres": {
+ "cpu_seconds": 1.5768600000000035,
+ "peak_sampled_anonymous_bytes": 14389248,
+ "peak_sampled_working_set_bytes": 87543808,
+ "rx_bytes": 1181342
+ },
+ "prometheus": {
+ "cpu_seconds": 0.03631899999999999,
+ "peak_sampled_anonymous_bytes": 31588352,
+ "peak_sampled_working_set_bytes": 44494848,
+ "rx_bytes": 90971
+ },
+ "redis": {
+ "cpu_seconds": 0.1754850000000001,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14110720,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.4327329999999998,
+ "peak_sampled_anonymous_bytes": 70475776,
+ "peak_sampled_working_set_bytes": 76066816,
+ "rx_bytes": 11142662
+ },
+ "s3-store": {
+ "cpu_seconds": 0.3934250000000006,
+ "peak_sampled_anonymous_bytes": 159215616,
+ "peak_sampled_working_set_bytes": 264777728,
+ "rx_bytes": 7488042
+ },
+ "tempo": {
+ "cpu_seconds": 0.10833899999999996,
+ "peak_sampled_anonymous_bytes": 166371328,
+ "peak_sampled_working_set_bytes": 168288256,
+ "rx_bytes": 35696
+ },
+ "worker": {
+ "cpu_seconds": 5.127224999999999,
+ "peak_sampled_anonymous_bytes": 78036992,
+ "peak_sampled_working_set_bytes": 82710528,
+ "rx_bytes": 3760839
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 441.0,
+ "otelcol_receiver_accepted_log_records_total": 840.0,
+ "otelcol_receiver_accepted_metric_points_total": 956.0,
+ "otelcol_exporter_sent_spans_total": 441.0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 441.0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "a039b96a91724972a86115bde3234c3a",
+ "status": 503,
+ "latency_seconds": 1.493530667000016,
+ "trace_sampled": true,
+ "trace_id": "5ed8acb1f3d78ada379de7c77298115c"
+ },
+ {
+ "request_id": "5dbafb887e7545179be21477bc5038f6",
+ "status": 200,
+ "latency_seconds": 0.0059597919999987425,
+ "trace_sampled": false,
+ "trace_id": "80eee5c8e270a673405e52b07e79ef4c"
+ },
+ {
+ "request_id": "5f84c3bb362d4eae9fa885ecd5c26dd9",
+ "status": 503,
+ "latency_seconds": 1.4955291660000114,
+ "trace_sampled": false,
+ "trace_id": "7ad13402acb0d2ba498fc8a34458d54d"
+ },
+ {
+ "request_id": "ed34bc1ab26b4dd8857ded5854ce7e49",
+ "status": 200,
+ "latency_seconds": 0.5992442920000087,
+ "trace_sampled": false,
+ "trace_id": "3ad0b8c978d2576caa4642e1dfe51a18"
+ }
+ ]
+ },
+ {
+ "mode": "full",
+ "enabled": 1,
+ "sample_ratio": 1,
+ "repeat": 1,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.971833499999974,
+ "successful_rps": 30.042309335294668,
+ "p50_seconds": 0.002800540999999157,
+ "p95_seconds": 0.008718707999975095,
+ "p99_seconds": 0.014413875000002463,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.640132,
+ "peak_sampled_anonymous_bytes": 167768064,
+ "peak_sampled_working_set_bytes": 176771072,
+ "rx_bytes": 4577482
+ },
+ "broker": {
+ "cpu_seconds": 0.14702400000000004,
+ "peak_sampled_anonymous_bytes": 3235840,
+ "peak_sampled_working_set_bytes": 8257536,
+ "rx_bytes": 45750
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.770845,
+ "peak_sampled_anonymous_bytes": 45436928,
+ "peak_sampled_working_set_bytes": 48349184,
+ "rx_bytes": 474731
+ },
+ "grafana": {
+ "cpu_seconds": 0.040907999999999944,
+ "peak_sampled_anonymous_bytes": 84176896,
+ "peak_sampled_working_set_bytes": 90267648,
+ "rx_bytes": 84
+ },
+ "lb": {
+ "cpu_seconds": 0.22938800000000015,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18636800,
+ "rx_bytes": 4283674
+ },
+ "loki": {
+ "cpu_seconds": 0.152698,
+ "peak_sampled_anonymous_bytes": 88342528,
+ "peak_sampled_working_set_bytes": 89530368,
+ "rx_bytes": 109077
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.182226,
+ "peak_sampled_anonymous_bytes": 61603840,
+ "peak_sampled_working_set_bytes": 67334144,
+ "rx_bytes": 1084156
+ },
+ "postgres": {
+ "cpu_seconds": 1.5757549999999938,
+ "peak_sampled_anonymous_bytes": 14557184,
+ "peak_sampled_working_set_bytes": 87969792,
+ "rx_bytes": 1185855
+ },
+ "prometheus": {
+ "cpu_seconds": 0.03547000000000011,
+ "peak_sampled_anonymous_bytes": 33968128,
+ "peak_sampled_working_set_bytes": 46882816,
+ "rx_bytes": 88548
+ },
+ "redis": {
+ "cpu_seconds": 0.16059299999999954,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14110720,
+ "rx_bytes": 119760
+ },
+ "s3": {
+ "cpu_seconds": 0.42028,
+ "peak_sampled_anonymous_bytes": 70524928,
+ "peak_sampled_working_set_bytes": 76115968,
+ "rx_bytes": 11142331
+ },
+ "s3-store": {
+ "cpu_seconds": 0.385713,
+ "peak_sampled_anonymous_bytes": 160899072,
+ "peak_sampled_working_set_bytes": 266690560,
+ "rx_bytes": 7487610
+ },
+ "tempo": {
+ "cpu_seconds": 0.1287790000000001,
+ "peak_sampled_anonymous_bytes": 175345664,
+ "peak_sampled_working_set_bytes": 177360896,
+ "rx_bytes": 90092
+ },
+ "worker": {
+ "cpu_seconds": 5.233713,
+ "peak_sampled_anonymous_bytes": 78712832,
+ "peak_sampled_working_set_bytes": 83640320,
+ "rx_bytes": 3767596
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 1950.0,
+ "otelcol_receiver_accepted_log_records_total": 840.0,
+ "otelcol_receiver_accepted_metric_points_total": 968.0,
+ "otelcol_exporter_sent_spans_total": 1950.0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 1950.0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "33a10f26c5144d20a848d29c7150b8e4",
+ "status": 200,
+ "latency_seconds": 0.005709916999990128,
+ "trace_sampled": true,
+ "trace_id": "80caa7fccdcaedd3cbe374cb0d38539e"
+ },
+ {
+ "request_id": "da630ecd36cb4471b9b36d9d3efee4c9",
+ "status": 503,
+ "latency_seconds": 1.6010425839999982,
+ "trace_sampled": true,
+ "trace_id": "d46dfa832b9aef901d583d1835729ea5"
+ },
+ {
+ "request_id": "a40b3ae805114393b29084b2ba9d4efd",
+ "status": 200,
+ "latency_seconds": 0.006991167000023779,
+ "trace_sampled": true,
+ "trace_id": "1af657ca6ff44d6d5bfc858fe2952c73"
+ },
+ {
+ "request_id": "4afb5c4b242944d5bdb1ca1c31878594",
+ "status": 200,
+ "latency_seconds": 0.006187374999996109,
+ "trace_sampled": true,
+ "trace_id": "e1b1f1d0d446379cc7807fc0b45b89e0"
+ }
+ ]
+ },
+ {
+ "mode": "sample10",
+ "enabled": 1,
+ "sample_ratio": 0.1,
+ "repeat": 1,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.969775749999997,
+ "successful_rps": 30.04540499159086,
+ "p50_seconds": 0.002478499999995165,
+ "p95_seconds": 0.009247708000032162,
+ "p99_seconds": 0.012284707999981492,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.549156,
+ "peak_sampled_anonymous_bytes": 184029184,
+ "peak_sampled_working_set_bytes": 193073152,
+ "rx_bytes": 4573747
+ },
+ "broker": {
+ "cpu_seconds": 0.1458149999999998,
+ "peak_sampled_anonymous_bytes": 3235840,
+ "peak_sampled_working_set_bytes": 8257536,
+ "rx_bytes": 46194
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.6954319999999999,
+ "peak_sampled_anonymous_bytes": 45428736,
+ "peak_sampled_working_set_bytes": 47878144,
+ "rx_bytes": 475787
+ },
+ "grafana": {
+ "cpu_seconds": 0.04330299999999987,
+ "peak_sampled_anonymous_bytes": 84189184,
+ "peak_sampled_working_set_bytes": 90279936,
+ "rx_bytes": 84
+ },
+ "lb": {
+ "cpu_seconds": 0.22905600000000081,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18382848,
+ "rx_bytes": 4285770
+ },
+ "loki": {
+ "cpu_seconds": 0.15075600000000033,
+ "peak_sampled_anonymous_bytes": 90304512,
+ "peak_sampled_working_set_bytes": 91512832,
+ "rx_bytes": 110502
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.17933100000000035,
+ "peak_sampled_anonymous_bytes": 61636608,
+ "peak_sampled_working_set_bytes": 67366912,
+ "rx_bytes": 766338
+ },
+ "postgres": {
+ "cpu_seconds": 1.4643060000000006,
+ "peak_sampled_anonymous_bytes": 15458304,
+ "peak_sampled_working_set_bytes": 89055232,
+ "rx_bytes": 1196312
+ },
+ "prometheus": {
+ "cpu_seconds": 0.03083099999999983,
+ "peak_sampled_anonymous_bytes": 34013184,
+ "peak_sampled_working_set_bytes": 46931968,
+ "rx_bytes": 88528
+ },
+ "redis": {
+ "cpu_seconds": 0.16615800000000025,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14114816,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.45296800000000026,
+ "peak_sampled_anonymous_bytes": 70594560,
+ "peak_sampled_working_set_bytes": 76312576,
+ "rx_bytes": 11144351
+ },
+ "s3-store": {
+ "cpu_seconds": 0.41154899999999905,
+ "peak_sampled_anonymous_bytes": 169603072,
+ "peak_sampled_working_set_bytes": 275623936,
+ "rx_bytes": 7489492
+ },
+ "tempo": {
+ "cpu_seconds": 0.18866799999999984,
+ "peak_sampled_anonymous_bytes": 263524352,
+ "peak_sampled_working_set_bytes": 265744384,
+ "rx_bytes": 27438
+ },
+ "worker": {
+ "cpu_seconds": 5.451653,
+ "peak_sampled_anonymous_bytes": 78204928,
+ "peak_sampled_working_set_bytes": 82956288,
+ "rx_bytes": 3761278
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 253.0,
+ "otelcol_receiver_accepted_log_records_total": 840.0,
+ "otelcol_receiver_accepted_metric_points_total": 987.0,
+ "otelcol_exporter_sent_spans_total": 253.0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 253.0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "f855f14c71f446b4b1b05738ba4a3548",
+ "status": 200,
+ "latency_seconds": 0.00400887499995406,
+ "trace_sampled": false,
+ "trace_id": "45fd1ec22cb5838e2ac74ab2a8bcffd2"
+ },
+ {
+ "request_id": "421dc6f8e82d4df79851e9edf53a9db2",
+ "status": 200,
+ "latency_seconds": 0.0038054579999879934,
+ "trace_sampled": false,
+ "trace_id": "58884c774148e2528053163d80eaca36"
+ },
+ {
+ "request_id": "fabe720f47f44d6d8cdd6aff17fa52da",
+ "status": 200,
+ "latency_seconds": 0.00652137499997707,
+ "trace_sampled": false,
+ "trace_id": "fee34bd08205389767462e2c7b7549a8"
+ },
+ {
+ "request_id": "8240a329f3f741ff9ea0f577f2820c39",
+ "status": 200,
+ "latency_seconds": 0.005179457999986425,
+ "trace_sampled": false,
+ "trace_id": "a8a3932c6d2b501a566bb82811d56f85"
+ }
+ ]
+ },
+ {
+ "mode": "sample25",
+ "enabled": 1,
+ "sample_ratio": 0.25,
+ "repeat": 1,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.97339687499999,
+ "successful_rps": 30.039957837667522,
+ "p50_seconds": 0.003318040999999994,
+ "p95_seconds": 0.011264500000038424,
+ "p99_seconds": 0.015699083000015435,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.7679620000000003,
+ "peak_sampled_anonymous_bytes": 154787840,
+ "peak_sampled_working_set_bytes": 163663872,
+ "rx_bytes": 4576911
+ },
+ "broker": {
+ "cpu_seconds": 0.17276000000000025,
+ "peak_sampled_anonymous_bytes": 3235840,
+ "peak_sampled_working_set_bytes": 8515584,
+ "rx_bytes": 45750
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.830878,
+ "peak_sampled_anonymous_bytes": 45432832,
+ "peak_sampled_working_set_bytes": 47878144,
+ "rx_bytes": 473183
+ },
+ "grafana": {
+ "cpu_seconds": 0.0578350000000003,
+ "peak_sampled_anonymous_bytes": 84205568,
+ "peak_sampled_working_set_bytes": 90296320,
+ "rx_bytes": 84
+ },
+ "lb": {
+ "cpu_seconds": 0.27045900000000067,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18382848,
+ "rx_bytes": 4285746
+ },
+ "loki": {
+ "cpu_seconds": 0.19686899999999952,
+ "peak_sampled_anonymous_bytes": 94715904,
+ "peak_sampled_working_set_bytes": 95989760,
+ "rx_bytes": 109961
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.2040059999999997,
+ "peak_sampled_anonymous_bytes": 61665280,
+ "peak_sampled_working_set_bytes": 67395584,
+ "rx_bytes": 822103
+ },
+ "postgres": {
+ "cpu_seconds": 1.7547759999999997,
+ "peak_sampled_anonymous_bytes": 15122432,
+ "peak_sampled_working_set_bytes": 88715264,
+ "rx_bytes": 1163538
+ },
+ "prometheus": {
+ "cpu_seconds": 0.03824000000000005,
+ "peak_sampled_anonymous_bytes": 35225600,
+ "peak_sampled_working_set_bytes": 48283648,
+ "rx_bytes": 87631
+ },
+ "redis": {
+ "cpu_seconds": 0.1955410000000004,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14110720,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.4847110000000008,
+ "peak_sampled_anonymous_bytes": 70668288,
+ "peak_sampled_working_set_bytes": 76337152,
+ "rx_bytes": 11145195
+ },
+ "s3-store": {
+ "cpu_seconds": 0.45569099999999985,
+ "peak_sampled_anonymous_bytes": 171560960,
+ "peak_sampled_working_set_bytes": 277803008,
+ "rx_bytes": 7486022
+ },
+ "tempo": {
+ "cpu_seconds": 0.07955000000000001,
+ "peak_sampled_anonymous_bytes": 337682432,
+ "peak_sampled_working_set_bytes": 339968000,
+ "rx_bytes": 35437
+ },
+ "worker": {
+ "cpu_seconds": 5.590355,
+ "peak_sampled_anonymous_bytes": 79478784,
+ "peak_sampled_working_set_bytes": 84197376,
+ "rx_bytes": 3762205
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 467.0,
+ "otelcol_receiver_accepted_log_records_total": 840.0,
+ "otelcol_receiver_accepted_metric_points_total": 1011.0,
+ "otelcol_exporter_sent_spans_total": 467.0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 467.0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "3e150ed6b6524aa7bd9038d8273671fd",
+ "status": 200,
+ "latency_seconds": 0.003437415999997029,
+ "trace_sampled": false,
+ "trace_id": "8a355908f3f46b435621829b8f061616"
+ },
+ {
+ "request_id": "310faf864ab841da8a15755c595d0009",
+ "status": 200,
+ "latency_seconds": 0.006516916999999012,
+ "trace_sampled": false,
+ "trace_id": "8f1be5ae6bb8a76ac59f1f2b37c5a821"
+ },
+ {
+ "request_id": "cc7ee3924c61494e91ae741593076c31",
+ "status": 200,
+ "latency_seconds": 0.006249749999994947,
+ "trace_sampled": true,
+ "trace_id": "56253c1d0b0dc58822128ecd06e6b929"
+ },
+ {
+ "request_id": "f5dec9866b5e41a39d1a908f6315a77b",
+ "status": 200,
+ "latency_seconds": 0.007077667000032761,
+ "trace_sampled": false,
+ "trace_id": "ae7e598b3c16f4117e4fe4d97560f1ff"
+ }
+ ]
+ },
+ {
+ "mode": "off",
+ "enabled": 0,
+ "sample_ratio": 0,
+ "repeat": 1,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.974153833999992,
+ "successful_rps": 30.038819415653062,
+ "p50_seconds": 0.0031486670000049344,
+ "p95_seconds": 0.009980333000044084,
+ "p99_seconds": 0.015300541999977213,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.3422620000000003,
+ "peak_sampled_anonymous_bytes": 167350272,
+ "peak_sampled_working_set_bytes": 176160768,
+ "rx_bytes": 4532506
+ },
+ "broker": {
+ "cpu_seconds": 0.1653519999999995,
+ "peak_sampled_anonymous_bytes": 3239936,
+ "peak_sampled_working_set_bytes": 8261632,
+ "rx_bytes": 43146
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.7756369999999999,
+ "peak_sampled_anonymous_bytes": 44990464,
+ "peak_sampled_working_set_bytes": 47382528,
+ "rx_bytes": 454292
+ },
+ "grafana": {
+ "cpu_seconds": 0.08641399999999999,
+ "peak_sampled_anonymous_bytes": 84217856,
+ "peak_sampled_working_set_bytes": 90308608,
+ "rx_bytes": 84
+ },
+ "lb": {
+ "cpu_seconds": 0.25412699999999955,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18903040,
+ "rx_bytes": 4325890
+ },
+ "loki": {
+ "cpu_seconds": 0.12718199999999946,
+ "peak_sampled_anonymous_bytes": 94724096,
+ "peak_sampled_working_set_bytes": 96002048,
+ "rx_bytes": 216
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.053601000000000454,
+ "peak_sampled_anonymous_bytes": 61677568,
+ "peak_sampled_working_set_bytes": 67411968,
+ "rx_bytes": 13027
+ },
+ "postgres": {
+ "cpu_seconds": 1.7014229999999984,
+ "peak_sampled_anonymous_bytes": 14426112,
+ "peak_sampled_working_set_bytes": 88383488,
+ "rx_bytes": 1174789
+ },
+ "prometheus": {
+ "cpu_seconds": 0.031208000000000125,
+ "peak_sampled_anonymous_bytes": 35622912,
+ "peak_sampled_working_set_bytes": 48680960,
+ "rx_bytes": 56562
+ },
+ "redis": {
+ "cpu_seconds": 0.18207400000000007,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14110720,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.4662059999999997,
+ "peak_sampled_anonymous_bytes": 70725632,
+ "peak_sampled_working_set_bytes": 76394496,
+ "rx_bytes": 11145979
+ },
+ "s3-store": {
+ "cpu_seconds": 0.4462469999999996,
+ "peak_sampled_anonymous_bytes": 171909120,
+ "peak_sampled_working_set_bytes": 278372352,
+ "rx_bytes": 7487825
+ },
+ "tempo": {
+ "cpu_seconds": 0.08142799999999983,
+ "peak_sampled_anonymous_bytes": 337686528,
+ "peak_sampled_working_set_bytes": 339976192,
+ "rx_bytes": 216
+ },
+ "worker": {
+ "cpu_seconds": 5.463901,
+ "peak_sampled_anonymous_bytes": 77950976,
+ "peak_sampled_working_set_bytes": 82563072,
+ "rx_bytes": 3738917
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 0,
+ "otelcol_receiver_accepted_log_records_total": 0,
+ "otelcol_receiver_accepted_metric_points_total": 0,
+ "otelcol_exporter_sent_spans_total": 0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "27e16e0cdaf24b43975a44ea4f079322",
+ "status": 200,
+ "latency_seconds": 0.0035381669999878795,
+ "trace_sampled": false
+ },
+ {
+ "request_id": "17921941164347fa920222eff5f2c418",
+ "status": 200,
+ "latency_seconds": 0.005465167000011206,
+ "trace_sampled": false
+ },
+ {
+ "request_id": "fb6e9dbce9b1447dad2e87af05165740",
+ "status": 200,
+ "latency_seconds": 0.007936124999957883,
+ "trace_sampled": false
+ },
+ {
+ "request_id": "1d92099d682c40068040a9d159f6b3c9",
+ "status": 200,
+ "latency_seconds": 0.004734958000028655,
+ "trace_sampled": false
+ }
+ ]
+ },
+ {
+ "mode": "sample10",
+ "enabled": 1,
+ "sample_ratio": 0.1,
+ "repeat": 2,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.972574916999974,
+ "successful_rps": 30.04119411209721,
+ "p50_seconds": 0.0030326660000241645,
+ "p95_seconds": 0.009149458999957005,
+ "p99_seconds": 0.01591166600002225,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.540737,
+ "peak_sampled_anonymous_bytes": 167518208,
+ "peak_sampled_working_set_bytes": 176734208,
+ "rx_bytes": 4569983
+ },
+ "broker": {
+ "cpu_seconds": 0.15188999999999986,
+ "peak_sampled_anonymous_bytes": 3239936,
+ "peak_sampled_working_set_bytes": 8261632,
+ "rx_bytes": 45750
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.803947,
+ "peak_sampled_anonymous_bytes": 45387776,
+ "peak_sampled_working_set_bytes": 47824896,
+ "rx_bytes": 467293
+ },
+ "grafana": {
+ "cpu_seconds": 0.050420999999999605,
+ "peak_sampled_anonymous_bytes": 84238336,
+ "peak_sampled_working_set_bytes": 90329088,
+ "rx_bytes": 84
+ },
+ "lb": {
+ "cpu_seconds": 0.22421000000000024,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18382848,
+ "rx_bytes": 4285710
+ },
+ "loki": {
+ "cpu_seconds": 0.17452000000000023,
+ "peak_sampled_anonymous_bytes": 95764480,
+ "peak_sampled_working_set_bytes": 97017856,
+ "rx_bytes": 109112
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.16662400000000055,
+ "peak_sampled_anonymous_bytes": 61689856,
+ "peak_sampled_working_set_bytes": 67436544,
+ "rx_bytes": 718356
+ },
+ "postgres": {
+ "cpu_seconds": 1.5899460000000047,
+ "peak_sampled_anonymous_bytes": 15716352,
+ "peak_sampled_working_set_bytes": 90574848,
+ "rx_bytes": 1185143
+ },
+ "prometheus": {
+ "cpu_seconds": 0.03482199999999991,
+ "peak_sampled_anonymous_bytes": 35635200,
+ "peak_sampled_working_set_bytes": 48697344,
+ "rx_bytes": 66971
+ },
+ "redis": {
+ "cpu_seconds": 0.1688029999999996,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14110720,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.4196410000000004,
+ "peak_sampled_anonymous_bytes": 70754304,
+ "peak_sampled_working_set_bytes": 76423168,
+ "rx_bytes": 11144409
+ },
+ "s3-store": {
+ "cpu_seconds": 0.4007069999999988,
+ "peak_sampled_anonymous_bytes": 172060672,
+ "peak_sampled_working_set_bytes": 278761472,
+ "rx_bytes": 7488666
+ },
+ "tempo": {
+ "cpu_seconds": 0.06864700000000012,
+ "peak_sampled_anonymous_bytes": 326729728,
+ "peak_sampled_working_set_bytes": 329039872,
+ "rx_bytes": 20737
+ },
+ "worker": {
+ "cpu_seconds": 5.257816,
+ "peak_sampled_anonymous_bytes": 78102528,
+ "peak_sampled_working_set_bytes": 82845696,
+ "rx_bytes": 3759275
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 105.0,
+ "otelcol_receiver_accepted_log_records_total": 840.0,
+ "otelcol_receiver_accepted_metric_points_total": 987.0,
+ "otelcol_exporter_sent_spans_total": 105.0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 105.0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "4295191b0f894e9abdf28777e3f0d575",
+ "status": 200,
+ "latency_seconds": 0.004693334000023697,
+ "trace_sampled": false,
+ "trace_id": "f1a103b24252c5fff5da5c0702517685"
+ },
+ {
+ "request_id": "b3fbc6b2a5f24410b55442f6b5cbf9ce",
+ "status": 200,
+ "latency_seconds": 0.0035256669999625956,
+ "trace_sampled": false,
+ "trace_id": "e761a8c7d4a55df0bf5997229c4732b3"
+ },
+ {
+ "request_id": "70ddcacb1f464c4187afbebbfb6d7191",
+ "status": 200,
+ "latency_seconds": 0.006238959000029354,
+ "trace_sampled": false,
+ "trace_id": "aa42b036d88b126fc9ceae5d8cffd9b9"
+ },
+ {
+ "request_id": "93ebeafb168c4fe19503616652b80255",
+ "status": 200,
+ "latency_seconds": 0.004492083999991792,
+ "trace_sampled": false,
+ "trace_id": "0514b97e37ae3528eef9fb2d6b7ee668"
+ }
+ ]
+ },
+ {
+ "mode": "sample25",
+ "enabled": 1,
+ "sample_ratio": 0.25,
+ "repeat": 2,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.971592207999947,
+ "successful_rps": 30.0426722992902,
+ "p50_seconds": 0.0030581659999597832,
+ "p95_seconds": 0.010556667000003017,
+ "p99_seconds": 0.017249125000034837,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.6178429999999997,
+ "peak_sampled_anonymous_bytes": 154996736,
+ "peak_sampled_working_set_bytes": 163852288,
+ "rx_bytes": 4576897
+ },
+ "broker": {
+ "cpu_seconds": 0.16159400000000002,
+ "peak_sampled_anonymous_bytes": 3235840,
+ "peak_sampled_working_set_bytes": 8261632,
+ "rx_bytes": 46194
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.798959,
+ "peak_sampled_anonymous_bytes": 45436928,
+ "peak_sampled_working_set_bytes": 47874048,
+ "rx_bytes": 473111
+ },
+ "grafana": {
+ "cpu_seconds": 0.05419600000000013,
+ "peak_sampled_anonymous_bytes": 84267008,
+ "peak_sampled_working_set_bytes": 90603520,
+ "rx_bytes": 154
+ },
+ "lb": {
+ "cpu_seconds": 0.24280599999999986,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18382848,
+ "rx_bytes": 4283976
+ },
+ "loki": {
+ "cpu_seconds": 0.17146299999999925,
+ "peak_sampled_anonymous_bytes": 96968704,
+ "peak_sampled_working_set_bytes": 98242560,
+ "rx_bytes": 110007
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.18281499999999973,
+ "peak_sampled_anonymous_bytes": 58724352,
+ "peak_sampled_working_set_bytes": 64479232,
+ "rx_bytes": 829611
+ },
+ "postgres": {
+ "cpu_seconds": 1.730197000000004,
+ "peak_sampled_anonymous_bytes": 14524416,
+ "peak_sampled_working_set_bytes": 89919488,
+ "rx_bytes": 1188701
+ },
+ "prometheus": {
+ "cpu_seconds": 0.04491600000000018,
+ "peak_sampled_anonymous_bytes": 35741696,
+ "peak_sampled_working_set_bytes": 48803840,
+ "rx_bytes": 90008
+ },
+ "redis": {
+ "cpu_seconds": 0.183643,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14110720,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.43968200000000124,
+ "peak_sampled_anonymous_bytes": 70795264,
+ "peak_sampled_working_set_bytes": 76468224,
+ "rx_bytes": 11144093
+ },
+ "s3-store": {
+ "cpu_seconds": 0.4230999999999998,
+ "peak_sampled_anonymous_bytes": 180441088,
+ "peak_sampled_working_set_bytes": 287334400,
+ "rx_bytes": 7486228
+ },
+ "tempo": {
+ "cpu_seconds": 0.07636799999999999,
+ "peak_sampled_anonymous_bytes": 309518336,
+ "peak_sampled_working_set_bytes": 311894016,
+ "rx_bytes": 38180
+ },
+ "worker": {
+ "cpu_seconds": 5.2202720000000005,
+ "peak_sampled_anonymous_bytes": 77426688,
+ "peak_sampled_working_set_bytes": 82075648,
+ "rx_bytes": 3761824
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 514.0,
+ "otelcol_receiver_accepted_log_records_total": 840.0,
+ "otelcol_receiver_accepted_metric_points_total": 1011.0,
+ "otelcol_exporter_sent_spans_total": 514.0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 514.0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "5aed70c2cc5b453eac6cf7e0655d7a98",
+ "status": 200,
+ "latency_seconds": 0.003553083999975115,
+ "trace_sampled": false,
+ "trace_id": "4d16fcd86d565a8147d7240bbd7fca85"
+ },
+ {
+ "request_id": "789efeeb93514993a772ad0ec225c177",
+ "status": 200,
+ "latency_seconds": 0.004778250000015305,
+ "trace_sampled": true,
+ "trace_id": "90fd2e18de6896542370fb061363f27c"
+ },
+ {
+ "request_id": "afb7b3b965b341f592ea0444430759d6",
+ "status": 200,
+ "latency_seconds": 0.006446417000006477,
+ "trace_sampled": false,
+ "trace_id": "497716bd475a42c3e2efad6607ad72ba"
+ },
+ {
+ "request_id": "e59306bbbe304566a5d3ae2769afbf99",
+ "status": 200,
+ "latency_seconds": 0.003542833000039991,
+ "trace_sampled": true,
+ "trace_id": "454797400df8d92c1101f75f42367560"
+ }
+ ]
+ },
+ {
+ "mode": "off",
+ "enabled": 0,
+ "sample_ratio": 0,
+ "repeat": 2,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.975750833000006,
+ "successful_rps": 30.03641790569384,
+ "p50_seconds": 0.0030265419999864207,
+ "p95_seconds": 0.00981791700002077,
+ "p99_seconds": 0.01680349999998043,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.3706709999999998,
+ "peak_sampled_anonymous_bytes": 154271744,
+ "peak_sampled_working_set_bytes": 162893824,
+ "rx_bytes": 4527464
+ },
+ "broker": {
+ "cpu_seconds": 0.15782899999999955,
+ "peak_sampled_anonymous_bytes": 3239936,
+ "peak_sampled_working_set_bytes": 8265728,
+ "rx_bytes": 43590
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.761449,
+ "peak_sampled_anonymous_bytes": 44986368,
+ "peak_sampled_working_set_bytes": 47316992,
+ "rx_bytes": 458050
+ },
+ "grafana": {
+ "cpu_seconds": 0.06901800000000025,
+ "peak_sampled_anonymous_bytes": 84291584,
+ "peak_sampled_working_set_bytes": 90624000,
+ "rx_bytes": 84
+ },
+ "lb": {
+ "cpu_seconds": 0.24708500000000022,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18382848,
+ "rx_bytes": 4325322
+ },
+ "loki": {
+ "cpu_seconds": 0.11884099999999975,
+ "peak_sampled_anonymous_bytes": 99409920,
+ "peak_sampled_working_set_bytes": 100728832,
+ "rx_bytes": 216
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.07021799999999967,
+ "peak_sampled_anonymous_bytes": 58736640,
+ "peak_sampled_working_set_bytes": 64471040,
+ "rx_bytes": 13525
+ },
+ "postgres": {
+ "cpu_seconds": 1.7200399999999973,
+ "peak_sampled_anonymous_bytes": 16109568,
+ "peak_sampled_working_set_bytes": 91168768,
+ "rx_bytes": 1179316
+ },
+ "prometheus": {
+ "cpu_seconds": 0.03256900000000007,
+ "peak_sampled_anonymous_bytes": 35774464,
+ "peak_sampled_working_set_bytes": 48836608,
+ "rx_bytes": 58692
+ },
+ "redis": {
+ "cpu_seconds": 0.1770290000000001,
+ "peak_sampled_anonymous_bytes": 3096576,
+ "peak_sampled_working_set_bytes": 14110720,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.4793459999999996,
+ "peak_sampled_anonymous_bytes": 70852608,
+ "peak_sampled_working_set_bytes": 76521472,
+ "rx_bytes": 11143786
+ },
+ "s3-store": {
+ "cpu_seconds": 0.45127899999999954,
+ "peak_sampled_anonymous_bytes": 188768256,
+ "peak_sampled_working_set_bytes": 295899136,
+ "rx_bytes": 7488189
+ },
+ "tempo": {
+ "cpu_seconds": 0.09413999999999989,
+ "peak_sampled_anonymous_bytes": 309526528,
+ "peak_sampled_working_set_bytes": 311910400,
+ "rx_bytes": 216
+ },
+ "worker": {
+ "cpu_seconds": 5.5170889999999995,
+ "peak_sampled_anonymous_bytes": 79003648,
+ "peak_sampled_working_set_bytes": 83582976,
+ "rx_bytes": 3739025
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 0,
+ "otelcol_receiver_accepted_log_records_total": 0,
+ "otelcol_receiver_accepted_metric_points_total": 0,
+ "otelcol_exporter_sent_spans_total": 0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "1ab6a284391a435fbe4be5202efafc76",
+ "status": 200,
+ "latency_seconds": 0.003744709000045532,
+ "trace_sampled": false
+ },
+ {
+ "request_id": "75df49c26b53430ea8705f286b255300",
+ "status": 503,
+ "latency_seconds": 1.5000359590000016,
+ "trace_sampled": false
+ },
+ {
+ "request_id": "fed96c6057a34feea01a663da8e4b758",
+ "status": 200,
+ "latency_seconds": 0.004121458000099665,
+ "trace_sampled": false
+ },
+ {
+ "request_id": "86c63d0f7d354e408467c022854e40c7",
+ "status": 200,
+ "latency_seconds": 0.006441458000040257,
+ "trace_sampled": false
+ }
+ ]
+ },
+ {
+ "mode": "full",
+ "enabled": 1,
+ "sample_ratio": 1,
+ "repeat": 2,
+ "arrival_rps": 30,
+ "scheduled_requests": 600,
+ "confirmed_requests": 600,
+ "load_wall_seconds": 19.972731667000062,
+ "successful_rps": 30.040958342786418,
+ "p50_seconds": 0.0028679590000137978,
+ "p95_seconds": 0.009695250000049782,
+ "p99_seconds": 0.014552875000049426,
+ "errors": [],
+ "image_jobs_ready": 30,
+ "resource": {
+ "api": {
+ "cpu_seconds": 2.7994320000000004,
+ "peak_sampled_anonymous_bytes": 154492928,
+ "peak_sampled_working_set_bytes": 163278848,
+ "rx_bytes": 4575815
+ },
+ "broker": {
+ "cpu_seconds": 0.15614999999999934,
+ "peak_sampled_anonymous_bytes": 3239936,
+ "peak_sampled_working_set_bytes": 8265728,
+ "rx_bytes": 45750
+ },
+ "dispatcher": {
+ "cpu_seconds": 0.7933490000000001,
+ "peak_sampled_anonymous_bytes": 45432832,
+ "peak_sampled_working_set_bytes": 48099328,
+ "rx_bytes": 478640
+ },
+ "grafana": {
+ "cpu_seconds": 0.051911999999999736,
+ "peak_sampled_anonymous_bytes": 84291584,
+ "peak_sampled_working_set_bytes": 90624000,
+ "rx_bytes": 84
+ },
+ "lb": {
+ "cpu_seconds": 0.24583200000000005,
+ "peak_sampled_anonymous_bytes": 7077888,
+ "peak_sampled_working_set_bytes": 18382848,
+ "rx_bytes": 4286798
+ },
+ "loki": {
+ "cpu_seconds": 0.17122700000000002,
+ "peak_sampled_anonymous_bytes": 99434496,
+ "peak_sampled_working_set_bytes": 100769792,
+ "rx_bytes": 108684
+ },
+ "otel-collector": {
+ "cpu_seconds": 0.21939300000000017,
+ "peak_sampled_anonymous_bytes": 58748928,
+ "peak_sampled_working_set_bytes": 64872448,
+ "rx_bytes": 1093250
+ },
+ "postgres": {
+ "cpu_seconds": 1.6113169999999997,
+ "peak_sampled_anonymous_bytes": 16871424,
+ "peak_sampled_working_set_bytes": 92184576,
+ "rx_bytes": 1162157
+ },
+ "prometheus": {
+ "cpu_seconds": 0.035209999999999964,
+ "peak_sampled_anonymous_bytes": 35790848,
+ "peak_sampled_working_set_bytes": 48857088,
+ "rx_bytes": 67071
+ },
+ "redis": {
+ "cpu_seconds": 0.17342299999999966,
+ "peak_sampled_anonymous_bytes": 4210688,
+ "peak_sampled_working_set_bytes": 15237120,
+ "rx_bytes": 119826
+ },
+ "s3": {
+ "cpu_seconds": 0.448224999999999,
+ "peak_sampled_anonymous_bytes": 70877184,
+ "peak_sampled_working_set_bytes": 76546048,
+ "rx_bytes": 11142216
+ },
+ "s3-store": {
+ "cpu_seconds": 0.4562039999999996,
+ "peak_sampled_anonymous_bytes": 189849600,
+ "peak_sampled_working_set_bytes": 297209856,
+ "rx_bytes": 7485564
+ },
+ "tempo": {
+ "cpu_seconds": 0.09863200000000028,
+ "peak_sampled_anonymous_bytes": 285548544,
+ "peak_sampled_working_set_bytes": 287961088,
+ "rx_bytes": 90073
+ },
+ "worker": {
+ "cpu_seconds": 5.735841,
+ "peak_sampled_anonymous_bytes": 77631488,
+ "peak_sampled_working_set_bytes": 82354176,
+ "rx_bytes": 3768983
+ }
+ },
+ "collector_ingest": {
+ "otelcol_receiver_accepted_spans_total": 1950.0,
+ "otelcol_receiver_accepted_log_records_total": 840.0,
+ "otelcol_receiver_accepted_metric_points_total": 1011.0,
+ "otelcol_exporter_sent_spans_total": 1950.0,
+ "otelcol_exporter_send_failed_spans_total": 0
+ },
+ "sampled_spans_finished": 1950.0,
+ "sampled_spans_not_accepted_after_drain": 0,
+ "rare_probes": [
+ {
+ "request_id": "efc0e010f6dc4f5e8e48b180754faed4",
+ "status": 200,
+ "latency_seconds": 0.007384333000004517,
+ "trace_sampled": true,
+ "trace_id": "10b2b587431a1429e7d6388ef5bca125"
+ },
+ {
+ "request_id": "be460fb9c47742b990c27d0113a5a518",
+ "status": 503,
+ "latency_seconds": 1.4986071670000456,
+ "trace_sampled": true,
+ "trace_id": "575f0fd60e3a98910c2e7ac7cee7b64f"
+ },
+ {
+ "request_id": "d6bde79ae2ed4239af44220aec6ec864",
+ "status": 503,
+ "latency_seconds": 1.4987055000000282,
+ "trace_sampled": true,
+ "trace_id": "22891f0abafba21cecc670122c55fd71"
+ },
+ {
+ "request_id": "7117d80c4b3c432ea08bc5f890a5585c",
+ "status": 200,
+ "latency_seconds": 0.005082291999997324,
+ "trace_sampled": true,
+ "trace_id": "6d930e564c92b0cdb743ed893a366505"
+ }
+ ]
+ }
+ ]
+}
diff --git a/evidence/observability-cost.png b/evidence/observability-cost.png
new file mode 100644
index 0000000..72e8fa1
Binary files /dev/null and b/evidence/observability-cost.png differ
diff --git a/evidence/observability-cost.svg b/evidence/observability-cost.svg
new file mode 100644
index 0000000..cdc8cb4
--- /dev/null
+++ b/evidence/observability-cost.svg
@@ -0,0 +1,4371 @@
+
+
+
diff --git a/evidence/observability-proof.txt b/evidence/observability-proof.txt
new file mode 100644
index 0000000..332b187
--- /dev/null
+++ b/evidence/observability-proof.txt
@@ -0,0 +1,8 @@
+ Container mtusi-lab6-worker-1 Stopping
+ Container mtusi-lab6-worker-1 Stopped
+ Container mtusi-lab6-s3-1 Paused
+ Container mtusi-lab6-worker-1 Starting
+ Container mtusi-lab6-worker-1 Started
+ Container mtusi-lab6-s3-1 Unpaused
+ Container mtusi-lab6-worker-1 Running
+PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.
diff --git a/evidence/persistence.txt b/evidence/persistence.txt
new file mode 100644
index 0000000..4af7cfe
--- /dev/null
+++ b/evidence/persistence.txt
@@ -0,0 +1,3 @@
+python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18093"
+Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
+PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
diff --git a/evidence/pipeline-regression.json b/evidence/pipeline-regression.json
new file mode 100644
index 0000000..719c43e
--- /dev/null
+++ b/evidence/pipeline-regression.json
@@ -0,0 +1,119 @@
+{
+ "invariants": {
+ "commit_before_publish_recovered": true,
+ "publish_before_marker_duplicate_safe": true,
+ "crashes_before_after_side_effect_and_commit": true,
+ "invalid_envelope_observable_dlq": true,
+ "duplicate_and_reorder_no_second_effect": true,
+ "concurrent_duplicate_single_claim": true,
+ "stale_running_version_fenced": true,
+ "old_delivery_after_new_ready_no_overwrite": true,
+ "failed_registry_explicit_replay_and_ready_refusal": true,
+ "transient_backoff_exhaustion_then_replay": true,
+ "invalid_decode_permanent_failed": true,
+ "cleanup_preserves_current_and_fresh_failed": true,
+ "ready_staging_cleanup_idempotent": true
+ },
+ "faults": [
+ {
+ "point": "before_side_effect",
+ "exit_code": 77,
+ "state_at_crash": "processing",
+ "recovered": "ready",
+ "attempts": 2,
+ "result_objects": 4
+ },
+ {
+ "point": "after_original",
+ "exit_code": 77,
+ "state_at_crash": "processing",
+ "recovered": "ready",
+ "attempts": 2,
+ "result_objects": 4
+ },
+ {
+ "point": "after_objects",
+ "exit_code": 77,
+ "state_at_crash": "processing",
+ "recovered": "ready",
+ "attempts": 2,
+ "result_objects": 4
+ },
+ {
+ "point": "after_commit",
+ "exit_code": 77,
+ "state_at_crash": "ready",
+ "recovered": "ready",
+ "attempts": 1,
+ "result_objects": 4
+ }
+ ],
+ "retry": {
+ "transitions": [
+ {
+ "state": "queued",
+ "attempt": 1,
+ "error_code": "storage_unavailable"
+ },
+ {
+ "state": "failed",
+ "attempt": 2,
+ "error_code": "retry_exhausted"
+ }
+ ],
+ "seconds": 1.983,
+ "replayed_attempt": 3
+ },
+ "transforms": [
+ {
+ "kind": "alpha",
+ "size": [
+ 20,
+ 10
+ ],
+ "exif_removed": true
+ },
+ {
+ "kind": "exif",
+ "size": [
+ 256,
+ 160
+ ],
+ "exif_removed": true
+ },
+ {
+ "kind": "small",
+ "size": [
+ 5,
+ 3
+ ],
+ "exif_removed": true
+ }
+ ],
+ "cleanup": [
+ {
+ "apply": false,
+ "retention_seconds": 86400,
+ "counts": {
+ "ready_staging": 397,
+ "expired_staging": 0
+ }
+ },
+ {
+ "apply": true,
+ "retention_seconds": 86400,
+ "counts": {
+ "ready_staging": 397,
+ "expired_staging": 0
+ }
+ },
+ {
+ "apply": true,
+ "retention_seconds": 86400,
+ "counts": {
+ "ready_staging": 0,
+ "expired_staging": 0
+ }
+ }
+ ]
+}
diff --git a/evidence/restart-after.json b/evidence/restart-after.json
new file mode 100644
index 0000000..15f0ecd
--- /dev/null
+++ b/evidence/restart-after.json
@@ -0,0 +1,546 @@
+[
+ {
+ "container": "cf2cfedf29de75b2458ec532a42f264cdb7ef71435900a7ef1c478d9d6b2e61f",
+ "name": "/mtusi-lab6-api-1",
+ "service": "api",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:45:44.122480904Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "8080/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_key",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_secret",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/database_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/cache_url",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b",
+ "name": "/mtusi-lab6-api-2",
+ "service": "api",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:45:44.196103962Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "8080/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/database_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/cache_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_key",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_secret",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "13d6ed83fc36b28aa57fc7cf302d9624098934b1f9c6ec68254a2b1a5f95228d",
+ "name": "/mtusi-lab6-broker-1",
+ "service": "broker",
+ "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
+ "user": "",
+ "running": true,
+ "started_at": "2026-11-20T08:45:30.515512908Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 201326592,
+ "nano_cpus": 500000000,
+ "ports": {
+ "6379/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/data",
+ "name": "mtusi-lab6_broker_data"
+ }
+ ]
+ },
+ {
+ "container": "076e71ce04ff5adbc19738e63a8ed319ad6f645183e6e65e47aec7dd97afe926",
+ "name": "/mtusi-lab6-bucket-bootstrap-1",
+ "service": "bucket-bootstrap",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": false,
+ "started_at": "2026-11-20T08:45:43.045813539Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 1000000000,
+ "ports": {},
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_admin_key",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_admin_secret",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "b2de6e448b4102ff9f25833bf99cd5a9978947dae0c3e28e743c3ef18f58b943",
+ "name": "/mtusi-lab6-dispatcher-1",
+ "service": "dispatcher",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:45:43.627780758Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "8080/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/database_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/broker_url",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "54edeaec98aff44bfe34f8c103c33694187550a9b690cde7648017ad323d7310",
+ "name": "/mtusi-lab6-grafana-1",
+ "service": "grafana",
+ "image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d",
+ "user": "472",
+ "running": true,
+ "started_at": "2026-11-20T08:45:30.734589337Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 500000000,
+ "ports": {
+ "3000/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "13006"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/etc/grafana/provisioning",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/var/lib/grafana/dashboards",
+ "name": null
+ },
+ {
+ "type": "volume",
+ "destination": "/var/lib/grafana",
+ "name": "mtusi-lab6_grafana_data"
+ }
+ ]
+ },
+ {
+ "container": "7db8217af00aa7be6a7e3d19f656faf329598ef85bfc8fb2fc2d15ec0cf72c28",
+ "name": "/mtusi-lab6-lb-1",
+ "service": "lb",
+ "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
+ "user": "haproxy",
+ "running": true,
+ "started_at": "2026-11-20T08:45:49.83573792Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 67108864,
+ "nano_cpus": 500000000,
+ "ports": {
+ "8080/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "18093"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/usr/local/etc/haproxy/haproxy.cfg",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "712535508b37fe699452fb2a4ed62a005a8cfa7a14e2b9827a9c82cd4ec982c9",
+ "name": "/mtusi-lab6-loki-1",
+ "service": "loki",
+ "image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd",
+ "user": "10001",
+ "running": true,
+ "started_at": "2026-11-20T08:45:31.075549983Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 500000000,
+ "ports": {
+ "3100/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "13106"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/etc/loki/config.yaml",
+ "name": null
+ },
+ {
+ "type": "volume",
+ "destination": "/loki",
+ "name": "mtusi-lab6_logs_data"
+ }
+ ]
+ },
+ {
+ "container": "6374cc6d3e2b163264c18e1181ea5be5b53f365a5324686a908ea8f94861a6c7",
+ "name": "/mtusi-lab6-migrate-1",
+ "service": "migrate",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": false,
+ "started_at": "2026-11-20T08:45:42.539353205Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 1000000000,
+ "ports": {},
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/cache_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_key",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_secret",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/database_url",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "89012420d895c4c7c0d9fdaf7c1ff40c3a5f97dca279a0ef8cec0be3903dce42",
+ "name": "/mtusi-lab6-otel-collector-1",
+ "service": "otel-collector",
+ "image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:45:30.652417444Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 500000000,
+ "ports": {
+ "4317/tcp": null,
+ "4318/tcp": null,
+ "55679/tcp": null,
+ "8888/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "18896"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/etc/otel/config.yaml",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "2257079aebc7a27eda5607608d7c4e7df3d1a196eb2c65d7236a1b60f7466c91",
+ "name": "/mtusi-lab6-postgres-1",
+ "service": "postgres",
+ "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
+ "user": "",
+ "running": true,
+ "started_at": "2026-11-20T08:45:30.708099365Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 536870912,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "5432/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/var/lib/postgresql/data",
+ "name": "mtusi-lab6_postgres_data"
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/postgres_password",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "937df26581a582233f77674d7c66a0b7fbba5fffd2377977ebd13857cd510a66",
+ "name": "/mtusi-lab6-prometheus-1",
+ "service": "prometheus",
+ "image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996",
+ "user": "nobody",
+ "running": true,
+ "started_at": "2026-11-20T08:45:30.770516143Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 500000000,
+ "ports": {
+ "9090/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "19096"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/prometheus",
+ "name": "mtusi-lab6_metrics_data"
+ },
+ {
+ "type": "bind",
+ "destination": "/etc/prometheus/prometheus.yaml",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/etc/prometheus/alerts.yaml",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "cc255a9bdb8967c853de2a4f167fe269a1a4f004e0b88ab174c79397212fd2a4",
+ "name": "/mtusi-lab6-redis-1",
+ "service": "redis",
+ "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
+ "user": "",
+ "running": true,
+ "started_at": "2026-11-20T08:45:30.676898647Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 67108864,
+ "nano_cpus": 250000000,
+ "ports": {
+ "6379/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/data",
+ "name": "db7892dbab7e8fc10b9d4431ee816406d604a540d7068b70aef208767168a178"
+ }
+ ]
+ },
+ {
+ "container": "c4564618c7c266a67ed1f6a05989f8156ca42582a1860fe8d868fc758d18d4f3",
+ "name": "/mtusi-lab6-s3-1",
+ "service": "s3",
+ "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
+ "user": "",
+ "running": true,
+ "started_at": "2026-11-20T08:45:30.688051331Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 536870912,
+ "nano_cpus": 500000000,
+ "ports": {
+ "18080/tcp": null,
+ "18888/tcp": null,
+ "19333/tcp": null,
+ "7333/tcp": null,
+ "8080/tcp": null,
+ "8333/tcp": null,
+ "8888/tcp": null,
+ "9333/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_config",
+ "name": null
+ },
+ {
+ "type": "volume",
+ "destination": "/data",
+ "name": "3f039af87109799a85e80069eb418dd6c13b9b036ffc46fb0e6ed1bfc21407de"
+ }
+ ]
+ },
+ {
+ "container": "d3abacb0bdaf348f062e5ec77599ea8a0450872a17b450b190ba7bce78166129",
+ "name": "/mtusi-lab6-s3-store-1",
+ "service": "s3-store",
+ "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
+ "user": "",
+ "running": true,
+ "started_at": "2026-11-20T08:45:31.797812564Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 536870912,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "18080/tcp": null,
+ "18888/tcp": null,
+ "19333/tcp": null,
+ "7333/tcp": null,
+ "8080/tcp": null,
+ "8333/tcp": null,
+ "8888/tcp": null,
+ "9333/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/data",
+ "name": "mtusi-lab6_s3_data"
+ }
+ ]
+ },
+ {
+ "container": "9cfc7a5053725737ecfc90211eed31a0c97a15aeb63ca270b9596c4af69de808",
+ "name": "/mtusi-lab6-tempo-1",
+ "service": "tempo",
+ "image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:45:40.551529375Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 536870912,
+ "nano_cpus": 500000000,
+ "ports": {
+ "3200/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "13206"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/var/tempo",
+ "name": "mtusi-lab6_traces_data"
+ },
+ {
+ "type": "bind",
+ "destination": "/etc/tempo/config.yaml",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "ef14a69e994f9f18fdb3438a03a28a974a3fcfc564f2d0eedeff3949a8f9bb29",
+ "name": "/mtusi-lab6-worker-1",
+ "service": "worker",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:45:44.122215389Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 805306368,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "8080/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/database_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/broker_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_worker_key",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_worker_secret",
+ "name": null
+ }
+ ]
+ }
+]
diff --git a/evidence/restart-before.json b/evidence/restart-before.json
new file mode 100644
index 0000000..348770b
--- /dev/null
+++ b/evidence/restart-before.json
@@ -0,0 +1,546 @@
+[
+ {
+ "container": "cf2cfedf29de75b2458ec532a42f264cdb7ef71435900a7ef1c478d9d6b2e61f",
+ "name": "/mtusi-lab6-api-1",
+ "service": "api",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:43:28.014761078Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "8080/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_key",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_secret",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/database_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/cache_url",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b",
+ "name": "/mtusi-lab6-api-2",
+ "service": "api",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:43:28.105739751Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "8080/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/database_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/cache_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_key",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_secret",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "13d6ed83fc36b28aa57fc7cf302d9624098934b1f9c6ec68254a2b1a5f95228d",
+ "name": "/mtusi-lab6-broker-1",
+ "service": "broker",
+ "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
+ "user": "",
+ "running": true,
+ "started_at": "2026-11-20T08:20:26.890287569Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 201326592,
+ "nano_cpus": 500000000,
+ "ports": {
+ "6379/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/data",
+ "name": "mtusi-lab6_broker_data"
+ }
+ ]
+ },
+ {
+ "container": "076e71ce04ff5adbc19738e63a8ed319ad6f645183e6e65e47aec7dd97afe926",
+ "name": "/mtusi-lab6-bucket-bootstrap-1",
+ "service": "bucket-bootstrap",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": false,
+ "started_at": "2026-11-20T08:40:43.538584225Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 1000000000,
+ "ports": {},
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_admin_key",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_admin_secret",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "b2de6e448b4102ff9f25833bf99cd5a9978947dae0c3e28e743c3ef18f58b943",
+ "name": "/mtusi-lab6-dispatcher-1",
+ "service": "dispatcher",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:43:28.015064203Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "8080/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/database_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/broker_url",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "54edeaec98aff44bfe34f8c103c33694187550a9b690cde7648017ad323d7310",
+ "name": "/mtusi-lab6-grafana-1",
+ "service": "grafana",
+ "image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d",
+ "user": "472",
+ "running": true,
+ "started_at": "2026-11-20T08:21:11.514848908Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 500000000,
+ "ports": {
+ "3000/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "13006"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/var/lib/grafana",
+ "name": "mtusi-lab6_grafana_data"
+ },
+ {
+ "type": "bind",
+ "destination": "/etc/grafana/provisioning",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/var/lib/grafana/dashboards",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "7db8217af00aa7be6a7e3d19f656faf329598ef85bfc8fb2fc2d15ec0cf72c28",
+ "name": "/mtusi-lab6-lb-1",
+ "service": "lb",
+ "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
+ "user": "haproxy",
+ "running": true,
+ "started_at": "2026-11-20T08:20:40.887085261Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 67108864,
+ "nano_cpus": 500000000,
+ "ports": {
+ "8080/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "18093"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/usr/local/etc/haproxy/haproxy.cfg",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "712535508b37fe699452fb2a4ed62a005a8cfa7a14e2b9827a9c82cd4ec982c9",
+ "name": "/mtusi-lab6-loki-1",
+ "service": "loki",
+ "image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd",
+ "user": "10001",
+ "running": true,
+ "started_at": "2026-11-20T08:21:11.513864698Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 500000000,
+ "ports": {
+ "3100/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "13106"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/etc/loki/config.yaml",
+ "name": null
+ },
+ {
+ "type": "volume",
+ "destination": "/loki",
+ "name": "mtusi-lab6_logs_data"
+ }
+ ]
+ },
+ {
+ "container": "6374cc6d3e2b163264c18e1181ea5be5b53f365a5324686a908ea8f94861a6c7",
+ "name": "/mtusi-lab6-migrate-1",
+ "service": "migrate",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": false,
+ "started_at": "2026-11-20T08:40:43.038416658Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 1000000000,
+ "ports": {},
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/database_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/cache_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_key",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_api_secret",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "89012420d895c4c7c0d9fdaf7c1ff40c3a5f97dca279a0ef8cec0be3903dce42",
+ "name": "/mtusi-lab6-otel-collector-1",
+ "service": "otel-collector",
+ "image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:21:11.514654968Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 500000000,
+ "ports": {
+ "4317/tcp": null,
+ "4318/tcp": null,
+ "55679/tcp": null,
+ "8888/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "18896"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/etc/otel/config.yaml",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "2257079aebc7a27eda5607608d7c4e7df3d1a196eb2c65d7236a1b60f7466c91",
+ "name": "/mtusi-lab6-postgres-1",
+ "service": "postgres",
+ "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
+ "user": "",
+ "running": true,
+ "started_at": "2026-11-20T08:20:26.86209481Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 536870912,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "5432/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/var/lib/postgresql/data",
+ "name": "mtusi-lab6_postgres_data"
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/postgres_password",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "937df26581a582233f77674d7c66a0b7fbba5fffd2377977ebd13857cd510a66",
+ "name": "/mtusi-lab6-prometheus-1",
+ "service": "prometheus",
+ "image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996",
+ "user": "nobody",
+ "running": true,
+ "started_at": "2026-11-20T08:21:11.515853797Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 268435456,
+ "nano_cpus": 500000000,
+ "ports": {
+ "9090/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "19096"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/prometheus",
+ "name": "mtusi-lab6_metrics_data"
+ },
+ {
+ "type": "bind",
+ "destination": "/etc/prometheus/prometheus.yaml",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/etc/prometheus/alerts.yaml",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "cc255a9bdb8967c853de2a4f167fe269a1a4f004e0b88ab174c79397212fd2a4",
+ "name": "/mtusi-lab6-redis-1",
+ "service": "redis",
+ "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
+ "user": "",
+ "running": true,
+ "started_at": "2026-11-20T08:20:26.875269419Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 67108864,
+ "nano_cpus": 250000000,
+ "ports": {
+ "6379/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/data",
+ "name": "db7892dbab7e8fc10b9d4431ee816406d604a540d7068b70aef208767168a178"
+ }
+ ]
+ },
+ {
+ "container": "c4564618c7c266a67ed1f6a05989f8156ca42582a1860fe8d868fc758d18d4f3",
+ "name": "/mtusi-lab6-s3-1",
+ "service": "s3",
+ "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
+ "user": "",
+ "running": true,
+ "started_at": "2026-11-20T08:20:31.57071865Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 536870912,
+ "nano_cpus": 500000000,
+ "ports": {
+ "18080/tcp": null,
+ "18888/tcp": null,
+ "19333/tcp": null,
+ "7333/tcp": null,
+ "8080/tcp": null,
+ "8333/tcp": null,
+ "8888/tcp": null,
+ "9333/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_config",
+ "name": null
+ },
+ {
+ "type": "volume",
+ "destination": "/data",
+ "name": "3f039af87109799a85e80069eb418dd6c13b9b036ffc46fb0e6ed1bfc21407de"
+ }
+ ]
+ },
+ {
+ "container": "d3abacb0bdaf348f062e5ec77599ea8a0450872a17b450b190ba7bce78166129",
+ "name": "/mtusi-lab6-s3-store-1",
+ "service": "s3-store",
+ "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
+ "user": "",
+ "running": true,
+ "started_at": "2026-11-20T08:20:26.891493875Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 536870912,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "18080/tcp": null,
+ "18888/tcp": null,
+ "19333/tcp": null,
+ "7333/tcp": null,
+ "8080/tcp": null,
+ "8333/tcp": null,
+ "8888/tcp": null,
+ "9333/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/data",
+ "name": "mtusi-lab6_s3_data"
+ }
+ ]
+ },
+ {
+ "container": "9cfc7a5053725737ecfc90211eed31a0c97a15aeb63ca270b9596c4af69de808",
+ "name": "/mtusi-lab6-tempo-1",
+ "service": "tempo",
+ "image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:28:48.715669396Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 536870912,
+ "nano_cpus": 500000000,
+ "ports": {
+ "3200/tcp": [
+ {
+ "HostIp": "127.0.0.1",
+ "HostPort": "13206"
+ }
+ ]
+ },
+ "mounts": [
+ {
+ "type": "volume",
+ "destination": "/var/tempo",
+ "name": "mtusi-lab6_traces_data"
+ },
+ {
+ "type": "bind",
+ "destination": "/etc/tempo/config.yaml",
+ "name": null
+ }
+ ]
+ },
+ {
+ "container": "ef14a69e994f9f18fdb3438a03a28a974a3fcfc564f2d0eedeff3949a8f9bb29",
+ "name": "/mtusi-lab6-worker-1",
+ "service": "worker",
+ "image": "mtusi-messenger:lab6",
+ "user": "10001:10001",
+ "running": true,
+ "started_at": "2026-11-20T08:45:08.489572558Z",
+ "exit_code": 0,
+ "oom_killed": false,
+ "memory_limit": 805306368,
+ "nano_cpus": 1000000000,
+ "ports": {
+ "8080/tcp": null
+ },
+ "mounts": [
+ {
+ "type": "bind",
+ "destination": "/run/secrets/database_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/broker_url",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_worker_key",
+ "name": null
+ },
+ {
+ "type": "bind",
+ "destination": "/run/secrets/s3_worker_secret",
+ "name": null
+ }
+ ]
+ }
+]
diff --git a/evidence/retry-trace.json b/evidence/retry-trace.json
new file mode 100644
index 0000000..0dc60b1
--- /dev/null
+++ b/evidence/retry-trace.json
@@ -0,0 +1,989 @@
+{
+ "trace_id": "dd1cb3e17dc547bc8bd3f94033e39f7a",
+ "retry_observed": true,
+ "trace": {
+ "batches": [
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "067e0d5099614580ab61516259b76c14"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-api"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "k4UL1Qk8WSI=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "name": "POST /api/v1/chats/{chat_id}/attachments",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795164308161361378",
+ "endTimeUnixNano": "1795164308169243229",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "POST"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/chats/{chat_id}/attachments"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "202"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "wgvjepi5Fj0=",
+ "parentSpanId": "k4UL1Qk8WSI=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164308161418698",
+ "endTimeUnixNano": "1795164308162079443",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "6f3FdmFjcfY=",
+ "parentSpanId": "k4UL1Qk8WSI=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164308162156009",
+ "endTimeUnixNano": "1795164308162662788",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "rvhnrnpClkQ=",
+ "parentSpanId": "k4UL1Qk8WSI=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164308163083649",
+ "endTimeUnixNano": "1795164308164393283",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "qBl9xvUTfq8=",
+ "parentSpanId": "k4UL1Qk8WSI=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164308164428226",
+ "endTimeUnixNano": "1795164308168921394",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "R2wwvDNpB68=",
+ "parentSpanId": "qBl9xvUTfq8=",
+ "name": "s3.put_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164308164676000",
+ "endTimeUnixNano": "1795164308168082844",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "put_object"
+ }
+ }
+ ],
+ "status": {}
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "6b019c7baab14c89976d2b4081f1c509"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-dispatcher"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "t4R0QxdjlSI=",
+ "parentSpanId": "qBl9xvUTfq8=",
+ "name": "attachment publish",
+ "kind": "SPAN_KIND_PRODUCER",
+ "startTimeUnixNano": "1795164308271913285",
+ "endTimeUnixNano": "1795164308274338175",
+ "attributes": [
+ {
+ "key": "messaging.system",
+ "value": {
+ "stringValue": "redis"
+ }
+ },
+ {
+ "key": "messaging.destination.name",
+ "value": {
+ "stringValue": "attachment-jobs"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "nBeLRAx4mXA=",
+ "parentSpanId": "qBl9xvUTfq8=",
+ "name": "attachment publish",
+ "kind": "SPAN_KIND_PRODUCER",
+ "startTimeUnixNano": "1795164312922355612",
+ "endTimeUnixNano": "1795164312924775467",
+ "attributes": [
+ {
+ "key": "messaging.system",
+ "value": {
+ "stringValue": "redis"
+ }
+ },
+ {
+ "key": "messaging.destination.name",
+ "value": {
+ "stringValue": "attachment-jobs"
+ }
+ }
+ ],
+ "status": {}
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "742be3a8d93342da9117f92711193f78"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-worker"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "AqkcRaZWl50=",
+ "parentSpanId": "t4R0QxdjlSI=",
+ "name": "attachment process",
+ "kind": "SPAN_KIND_CONSUMER",
+ "startTimeUnixNano": "1795164309031454501",
+ "endTimeUnixNano": "1795164312029923986",
+ "attributes": [
+ {
+ "key": "messaging.system",
+ "value": {
+ "stringValue": "redis"
+ }
+ },
+ {
+ "key": "messaging.destination.name",
+ "value": {
+ "stringValue": "attachment-jobs"
+ }
+ },
+ {
+ "key": "job.id",
+ "value": {
+ "stringValue": "9d711cdf-347d-4590-91da-4908beb37202"
+ }
+ },
+ {
+ "key": "job.attempt",
+ "value": {
+ "intValue": "1"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "cq34RtwooBc=",
+ "parentSpanId": "AqkcRaZWl50=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164309031484554",
+ "endTimeUnixNano": "1795164309038640365",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "PCx2LcCU6hc=",
+ "parentSpanId": "AqkcRaZWl50=",
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164309108621018",
+ "endTimeUnixNano": "1795164312013033788",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "ReadTimeoutError"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "cX5pEW5zHb0=",
+ "parentSpanId": "AqkcRaZWl50=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164312013553635",
+ "endTimeUnixNano": "1795164312029457465",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "yG0WSHlT7kQ=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164312924989622",
+ "endTimeUnixNano": "1795164312936812921",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-worker"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "742be3a8d93342da9117f92711193f78"
+ }
+ },
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "ccEcqG6ZxPo=",
+ "parentSpanId": "nBeLRAx4mXA=",
+ "flags": 768,
+ "name": "attachment process",
+ "kind": "SPAN_KIND_CONSUMER",
+ "startTimeUnixNano": "1795164312924913496",
+ "endTimeUnixNano": "1795164314747321271",
+ "attributes": [
+ {
+ "key": "messaging.system",
+ "value": {
+ "stringValue": "redis"
+ }
+ },
+ {
+ "key": "messaging.destination.name",
+ "value": {
+ "stringValue": "attachment-jobs"
+ }
+ },
+ {
+ "key": "job.id",
+ "value": {
+ "stringValue": "9d711cdf-347d-4590-91da-4908beb37202"
+ }
+ },
+ {
+ "key": "job.attempt",
+ "value": {
+ "intValue": "2"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "Qf833YZm6d8=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164312943026506",
+ "endTimeUnixNano": "1795164314534106374",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "7QJGzOLYHEk=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.get_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314534253963",
+ "endTimeUnixNano": "1795164314538495180",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "get_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "uj8SpqBZjXM=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314714992685",
+ "endTimeUnixNano": "1795164314719977680",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "ClientError"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "RPy3F6vP7SY=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.put_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314720094322",
+ "endTimeUnixNano": "1795164314725722348",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "put_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "HvB6dqnveCs=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314725791217",
+ "endTimeUnixNano": "1795164314727041344",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "ClientError"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "DJqV5J+noKw=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.put_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314727130718",
+ "endTimeUnixNano": "1795164314729233570",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "put_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "YhhL1bvMeTM=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314729272549",
+ "endTimeUnixNano": "1795164314730313956",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "ClientError"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "92rjoPWvm5g=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.put_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314730379104",
+ "endTimeUnixNano": "1795164314732329971",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "put_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "7VymNXkyejE=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314732386665",
+ "endTimeUnixNano": "1795164314733725616",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "ClientError"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "5RGC0O8dQC8=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.put_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314733807464",
+ "endTimeUnixNano": "1795164314736051098",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "put_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "EGXBlCQ2GhE=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314736092741",
+ "endTimeUnixNano": "1795164314743975478",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
+ "spanId": "uADsu2p9U4s=",
+ "parentSpanId": "ccEcqG6ZxPo=",
+ "flags": 256,
+ "name": "s3.delete_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164314744065993",
+ "endTimeUnixNano": "1795164314746751040",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "delete_object"
+ }
+ }
+ ],
+ "status": {}
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "logs": [
+ {
+ "timestamp": "2026-11-20T08:45:12.027960+00:00",
+ "severity": "INFO",
+ "service.name": "messenger-worker",
+ "event": "job_retry",
+ "sampled": true,
+ "trace_id": "dd1cb3e17dc547bc8bd3f94033e39f7a",
+ "span_id": "717e69116e731dbd",
+ "job_id": "9d711cdf-347d-4590-91da-4908beb37202",
+ "attempt": 1,
+ "error_code": "storage_unavailable"
+ }
+ ]
+}
diff --git a/evidence/sampling-retention.json b/evidence/sampling-retention.json
new file mode 100644
index 0000000..f126d72
--- /dev/null
+++ b/evidence/sampling-retention.json
@@ -0,0 +1,348 @@
+{
+ "method": "Three repeats per full/10%/25% setting; each holds sessions lock until pg_locks confirms it. 3x503 plus one slow200 per repeat. Sampling is ordinary SDK head sampling without forced traceparent.",
+ "runs": [
+ {
+ "mode": "full",
+ "repeat": 0,
+ "probes": [
+ {
+ "request_id": "95355e81cca54f83be9a19bd527e9825",
+ "status": 503,
+ "latency_seconds": 1.5030926670000007,
+ "trace_sampled": true,
+ "trace_id": "99c07cebfe6d37a278f4e0a9c603c1c2",
+ "tempo_trace_found": true
+ },
+ {
+ "request_id": "4e617be83d2247d0b849744fa0936deb",
+ "status": 503,
+ "latency_seconds": 1.6241221669999995,
+ "trace_sampled": true,
+ "trace_id": "806f738e3362110032113077494f8c8b",
+ "tempo_trace_found": true
+ },
+ {
+ "request_id": "b1c982a06765483eac67aa8cf72f2be1",
+ "status": 503,
+ "latency_seconds": 1.4874708749999996,
+ "trace_sampled": true,
+ "trace_id": "a4f07c609d396e2071bb8bdbebcbad4f",
+ "tempo_trace_found": true
+ },
+ {
+ "request_id": "822110e4f137423084355e5c8d260dcb",
+ "status": 200,
+ "latency_seconds": 1.1728969580000026,
+ "trace_sampled": true,
+ "trace_id": "8082c09a793e3535f33f203cb1dbe5da",
+ "tempo_trace_found": true
+ }
+ ]
+ },
+ {
+ "mode": "full",
+ "repeat": 1,
+ "probes": [
+ {
+ "request_id": "da8b813d32d144cd844124a5fa86435e",
+ "status": 503,
+ "latency_seconds": 1.4984728329999975,
+ "trace_sampled": true,
+ "trace_id": "0fe73d0ff759f1e9f2f6d9e304a8fb96",
+ "tempo_trace_found": true
+ },
+ {
+ "request_id": "afb0eccacc314dcc8d15e6375ed1f674",
+ "status": 503,
+ "latency_seconds": 1.5027760410000006,
+ "trace_sampled": true,
+ "trace_id": "04c97d704d04c30ed712dc1a6c506639",
+ "tempo_trace_found": true
+ },
+ {
+ "request_id": "2633bfaaa83040dc9ec3f21c968b6774",
+ "status": 503,
+ "latency_seconds": 1.6313028749999994,
+ "trace_sampled": true,
+ "trace_id": "a849e96eb0258348dba259183e9455b5",
+ "tempo_trace_found": true
+ },
+ {
+ "request_id": "5f1a8b638d594e7e8e249d83e29a3c0a",
+ "status": 200,
+ "latency_seconds": 1.1764352089999974,
+ "trace_sampled": true,
+ "trace_id": "46f17923d4d83be8aec4863497913de9",
+ "tempo_trace_found": true
+ }
+ ]
+ },
+ {
+ "mode": "full",
+ "repeat": 2,
+ "probes": [
+ {
+ "request_id": "f6cf9187274f4d4494f2546fdccf9cb5",
+ "status": 503,
+ "latency_seconds": 1.4932770829999953,
+ "trace_sampled": true,
+ "trace_id": "b57747b1e2c746aafa5fb7d7e9ec7e33",
+ "tempo_trace_found": true
+ },
+ {
+ "request_id": "5820d13764fe4ab5826a24e6be1dd12c",
+ "status": 503,
+ "latency_seconds": 1.5006451660000053,
+ "trace_sampled": true,
+ "trace_id": "f167d8d9df62e009ab0c2c3ac4ee3fc3",
+ "tempo_trace_found": true
+ },
+ {
+ "request_id": "18e12c0c4a424141b916fd4f7c847c8b",
+ "status": 503,
+ "latency_seconds": 1.5060892499999952,
+ "trace_sampled": true,
+ "trace_id": "453c386a3fcd16e940832672ef0b072a",
+ "tempo_trace_found": true
+ },
+ {
+ "request_id": "56df38255f974d75a68b6defb7ae7dbf",
+ "status": 200,
+ "latency_seconds": 1.181737209000005,
+ "trace_sampled": true,
+ "trace_id": "9380177eb739659ba6bfb122f4c5cb10",
+ "tempo_trace_found": true
+ }
+ ]
+ },
+ {
+ "mode": "sample10",
+ "repeat": 0,
+ "probes": [
+ {
+ "request_id": "7c5131770a4e4ae78594065d30041f6b",
+ "status": 503,
+ "latency_seconds": 1.5020687499999994,
+ "trace_sampled": false,
+ "trace_id": "2f49be17fa517e268808bc7655dfec57",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "083b4e2c54b34fb695e3e097774a02fb",
+ "status": 503,
+ "latency_seconds": 1.501452209000007,
+ "trace_sampled": false,
+ "trace_id": "6b7b3c7eb4f9d107cdef92d247489d39",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "d63a94e59cc74918881a6c4821129fc0",
+ "status": 503,
+ "latency_seconds": 1.5036532090000065,
+ "trace_sampled": false,
+ "trace_id": "e6b3c5a1434815bc391a764eba18ad1f",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "7aba69c8960a4e58ab8e1189f91e8df8",
+ "status": 200,
+ "latency_seconds": 1.1895337499999954,
+ "trace_sampled": true,
+ "trace_id": "a4b7c2e50c6db0db11b63d6988a5f581",
+ "tempo_trace_found": true
+ }
+ ]
+ },
+ {
+ "mode": "sample10",
+ "repeat": 1,
+ "probes": [
+ {
+ "request_id": "db9a19fe6edf44b189e4c85527305b97",
+ "status": 503,
+ "latency_seconds": 1.508617583000003,
+ "trace_sampled": false,
+ "trace_id": "43478ce0a70fc97c1f551e0b770c8450",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "259f18e925b54205a7f86ccf23588581",
+ "status": 503,
+ "latency_seconds": 1.6092364170000053,
+ "trace_sampled": false,
+ "trace_id": "de05c2c1da9da6e5aaad96f7562f8626",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "7463cb267c1b4b7c836a79772d11d304",
+ "status": 503,
+ "latency_seconds": 1.4921612919999916,
+ "trace_sampled": false,
+ "trace_id": "63f2de04875064912b355e6f7ccf2bf8",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "e6013482b7174a469182f9776832c5f8",
+ "status": 200,
+ "latency_seconds": 1.1786814999999962,
+ "trace_sampled": false,
+ "trace_id": "bc385f8114ac1d41fa55f762830d9a70",
+ "tempo_trace_found": false
+ }
+ ]
+ },
+ {
+ "mode": "sample10",
+ "repeat": 2,
+ "probes": [
+ {
+ "request_id": "9a8de42292644772a11b19bb597b98e9",
+ "status": 503,
+ "latency_seconds": 1.4827315419999962,
+ "trace_sampled": false,
+ "trace_id": "62ef4444e74027412566b41ff3330a09",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "444fd3c3d88c46cd89e1691852087fb3",
+ "status": 503,
+ "latency_seconds": 1.4932578330000013,
+ "trace_sampled": false,
+ "trace_id": "95d7610acd0f35487ec10e503fc12622",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "6f616e3942134f7797c6c0b3767c0d0e",
+ "status": 503,
+ "latency_seconds": 1.497054833000007,
+ "trace_sampled": false,
+ "trace_id": "86793b238c9e2b7e586cb02e2f8bfc42",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "718f724c1a8d41bfa3ec720daee432d3",
+ "status": 200,
+ "latency_seconds": 1.1874764580000061,
+ "trace_sampled": false,
+ "trace_id": "6b08c6339522e460bd77a9fd0f72438a",
+ "tempo_trace_found": false
+ }
+ ]
+ },
+ {
+ "mode": "sample25",
+ "repeat": 0,
+ "probes": [
+ {
+ "request_id": "5c805a3468b64739bd1c839f6ddedd56",
+ "status": 503,
+ "latency_seconds": 1.4879085000000032,
+ "trace_sampled": false,
+ "trace_id": "5cf9d06acb989768f56a26c31e6d5255",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "ff5596ad44c6461589e621a7b3f09676",
+ "status": 503,
+ "latency_seconds": 1.5042155419999972,
+ "trace_sampled": false,
+ "trace_id": "f8f790abc18c384d58225dceb4d1e8a9",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "3ee8c8db9d4a4890b866d3cb40e18bb8",
+ "status": 503,
+ "latency_seconds": 1.4965452500000112,
+ "trace_sampled": false,
+ "trace_id": "b85f2a7d3cd2ff3dd4fa66fc8b37dea6",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "f21c138335d64c688adb25c137c9a253",
+ "status": 200,
+ "latency_seconds": 1.1858503330000048,
+ "trace_sampled": true,
+ "trace_id": "c1df2d7ecfb8448a2f33be6756f23c38",
+ "tempo_trace_found": true
+ }
+ ]
+ },
+ {
+ "mode": "sample25",
+ "repeat": 1,
+ "probes": [
+ {
+ "request_id": "053ece24cfa1465d9702c67781863703",
+ "status": 503,
+ "latency_seconds": 1.5010194580000018,
+ "trace_sampled": true,
+ "trace_id": "b7c73109f24a628720011f7401fc3d42",
+ "tempo_trace_found": true
+ },
+ {
+ "request_id": "7d30c3b85ebe480ea041d2261af90b3c",
+ "status": 503,
+ "latency_seconds": 1.4840848329999972,
+ "trace_sampled": false,
+ "trace_id": "438e9ec5155e621e5ef24df2b6e12568",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "8c23bd6e2acc451d8abac7c787225b06",
+ "status": 503,
+ "latency_seconds": 1.4938515830000085,
+ "trace_sampled": false,
+ "trace_id": "92267b13ab39964ae58d8bf054c2f52a",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "3ddb46344bae4d50989c8779264eeadf",
+ "status": 200,
+ "latency_seconds": 1.1738840829999901,
+ "trace_sampled": false,
+ "trace_id": "66d1a465ebede54e734068bc6e6092ca",
+ "tempo_trace_found": false
+ }
+ ]
+ },
+ {
+ "mode": "sample25",
+ "repeat": 2,
+ "probes": [
+ {
+ "request_id": "dbcb7d27ff3a4067af49f2ad559beaf7",
+ "status": 503,
+ "latency_seconds": 1.4999495829999887,
+ "trace_sampled": false,
+ "trace_id": "88fa0b73db8d67b0be62f9ef653d5c1c",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "1644972bdbbf4ad4ae0bb1be96d60853",
+ "status": 503,
+ "latency_seconds": 1.5033828749999998,
+ "trace_sampled": false,
+ "trace_id": "815ba97d46bc980d6df03e8542f1bd94",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "34ecddeb28764f6da5aae311d8297f15",
+ "status": 503,
+ "latency_seconds": 1.486551832999993,
+ "trace_sampled": false,
+ "trace_id": "5b1d99439c3aadf44db722c9543be3f3",
+ "tempo_trace_found": false
+ },
+ {
+ "request_id": "558612f66b374e6d964b927a866bd9bf",
+ "status": 200,
+ "latency_seconds": 1.1901341250000144,
+ "trace_sampled": false,
+ "trace_id": "e6505908ffe05caf83a8a7d49f4476a0",
+ "tempo_trace_found": false
+ }
+ ]
+ }
+ ],
+ "presence_verification": "Actual Tempo trace API checked for every probe: sampled present; unsampled HTTP404."
+}
diff --git a/evidence/stopped-instances.jsonl b/evidence/stopped-instances.jsonl
new file mode 100644
index 0000000..8e4ba37
--- /dev/null
+++ b/evidence/stopped-instances.jsonl
@@ -0,0 +1 @@
+{"instance": "208b43d0e3fe4d8daeccd4d252d691fe", "container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b", "running": false, "exit_code": 0, "seconds": 0.691}
diff --git a/evidence/storage-security.json b/evidence/storage-security.json
new file mode 100644
index 0000000..c1b3f72
--- /dev/null
+++ b/evidence/storage-security.json
@@ -0,0 +1,16 @@
+{
+ "anonymous_object_and_list_403": true,
+ "job_fields": [
+ "attachment_id",
+ "attempt",
+ "created_at",
+ "job_id",
+ "pipeline_version",
+ "schema_version",
+ "traceparent",
+ "type"
+ ],
+ "job_bytes": 317,
+ "no_source_bytes_or_credentials_in_envelope": true,
+ "strict_invalid_envelope_checks": true
+}
diff --git a/evidence/telemetry-persistence.json b/evidence/telemetry-persistence.json
new file mode 100644
index 0000000..7e56e0d
--- /dev/null
+++ b/evidence/telemetry-persistence.json
@@ -0,0 +1,27 @@
+{
+ "trace_id": "a630912396b64101b293cc306ee44ed8",
+ "historical_span_count": 21,
+ "historical_logs": 8,
+ "fixed_metric_query_time": 1795164311.5720682,
+ "metric_before": [
+ {
+ "metric": {},
+ "value": [
+ 1795164311.572,
+ "485"
+ ]
+ }
+ ],
+ "metric_after": [
+ {
+ "metric": {},
+ "value": [
+ 1795164311.572,
+ "485"
+ ]
+ }
+ ],
+ "business_data_and_existing_session_survived": true,
+ "all_telemetry_volumes_restarted": true,
+ "trace_log_metric_history_survived": true
+}
diff --git a/evidence/trace-proof.json b/evidence/trace-proof.json
new file mode 100644
index 0000000..8f789cd
--- /dev/null
+++ b/evidence/trace-proof.json
@@ -0,0 +1,886 @@
+{
+ "trace_id": "a630912396b64101b293cc306ee44ed8",
+ "request_id": "cb06618bb089444e8958f45beb2a8d1f",
+ "status": "ready",
+ "consumer_parent_matches_producer": true,
+ "producer_descends_from_upload": true,
+ "api_worker_logs_correlated": true,
+ "secret_canaries_absent": true,
+ "metric_labels_allowlisted": true,
+ "trace": {
+ "batches": [
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-dispatcher"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "6b019c7baab14c89976d2b4081f1c509"
+ }
+ },
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "1X47WVGmwVA=",
+ "parentSpanId": "XoCVQvOw3eM=",
+ "flags": 768,
+ "name": "attachment publish",
+ "kind": "SPAN_KIND_PRODUCER",
+ "startTimeUnixNano": "1795164306352857288",
+ "endTimeUnixNano": "1795164306353855758",
+ "attributes": [
+ {
+ "key": "messaging.system",
+ "value": {
+ "stringValue": "redis"
+ }
+ },
+ {
+ "key": "messaging.destination.name",
+ "value": {
+ "stringValue": "attachment-jobs"
+ }
+ }
+ ],
+ "status": {}
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-worker"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "c474b80aef4f42b1ba83aa52fc227420"
+ }
+ },
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "jpUcOdrK9v8=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306353943301",
+ "endTimeUnixNano": "1795164306358874754",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "uqFoOGdo/j8=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306361469707",
+ "endTimeUnixNano": "1795164306363607190",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "Djfujh2+aLk=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.get_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306363643778",
+ "endTimeUnixNano": "1795164306365315189",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "get_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "UqSD5a1Hm0M=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306502165196",
+ "endTimeUnixNano": "1795164306504695870",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "ClientError"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "FIWbTD1UmKU=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.put_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306504774140",
+ "endTimeUnixNano": "1795164306509272099",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "put_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "sGmfWvZhAhE=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306509328359",
+ "endTimeUnixNano": "1795164306510325785",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "ClientError"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "cTvaoR1LX60=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.put_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306510386972",
+ "endTimeUnixNano": "1795164306512247503",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "put_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "n7+0IIgeeOo=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306512289102",
+ "endTimeUnixNano": "1795164306513273664",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "ClientError"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "z5ewZlRmlSI=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.put_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306513330090",
+ "endTimeUnixNano": "1795164306515021299",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "put_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "ThQkryD54tU=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.head_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306515051329",
+ "endTimeUnixNano": "1795164306515871498",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "head_object"
+ }
+ },
+ {
+ "key": "error.type",
+ "value": {
+ "stringValue": "ClientError"
+ }
+ }
+ ],
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "3F5tg5VjfcM=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.put_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306515917149",
+ "endTimeUnixNano": "1795164306517590816",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "put_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "jRNKytkgcSc=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306517629032",
+ "endTimeUnixNano": "1795164306523308482",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "EPrL86X8HKM=",
+ "parentSpanId": "wUZSkS75RVs=",
+ "flags": 256,
+ "name": "s3.delete_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306523346489",
+ "endTimeUnixNano": "1795164306524869630",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "delete_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "wUZSkS75RVs=",
+ "parentSpanId": "1X47WVGmwVA=",
+ "flags": 768,
+ "name": "attachment process",
+ "kind": "SPAN_KIND_CONSUMER",
+ "startTimeUnixNano": "1795164306353917740",
+ "endTimeUnixNano": "1795164306525244943",
+ "attributes": [
+ {
+ "key": "messaging.system",
+ "value": {
+ "stringValue": "redis"
+ }
+ },
+ {
+ "key": "messaging.destination.name",
+ "value": {
+ "stringValue": "attachment-jobs"
+ }
+ },
+ {
+ "key": "job.id",
+ "value": {
+ "stringValue": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2"
+ }
+ },
+ {
+ "key": "job.attempt",
+ "value": {
+ "intValue": "1"
+ }
+ }
+ ],
+ "status": {}
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "python"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "1.36.0"
+ }
+ },
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "messenger-api"
+ }
+ },
+ {
+ "key": "service.version",
+ "value": {
+ "stringValue": "lab6.1"
+ }
+ },
+ {
+ "key": "service.instance.id",
+ "value": {
+ "stringValue": "067e0d5099614580ab61516259b76c14"
+ }
+ },
+ {
+ "key": "deployment.environment.name",
+ "value": {
+ "stringValue": "lab"
+ }
+ }
+ ]
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "messenger"
+ },
+ "spans": [
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "s72qs5x19wo=",
+ "parentSpanId": "bWCElBq0RoM=",
+ "flags": 256,
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306279538296",
+ "endTimeUnixNano": "1795164306279912941",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "9hTiABG5UiA=",
+ "parentSpanId": "bWCElBq0RoM=",
+ "flags": 256,
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306279948610",
+ "endTimeUnixNano": "1795164306280391627",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "PPdidexZwfM=",
+ "parentSpanId": "bWCElBq0RoM=",
+ "flags": 256,
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306280710138",
+ "endTimeUnixNano": "1795164306281834242",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "PKeSMnEZomk=",
+ "parentSpanId": "XoCVQvOw3eM=",
+ "flags": 256,
+ "name": "s3.put_object",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306282049715",
+ "endTimeUnixNano": "1795164306285026998",
+ "attributes": [
+ {
+ "key": "rpc.system",
+ "value": {
+ "stringValue": "aws-api"
+ }
+ },
+ {
+ "key": "rpc.service",
+ "value": {
+ "stringValue": "S3"
+ }
+ },
+ {
+ "key": "rpc.method",
+ "value": {
+ "stringValue": "put_object"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "XoCVQvOw3eM=",
+ "parentSpanId": "bWCElBq0RoM=",
+ "flags": 256,
+ "name": "db.transaction",
+ "kind": "SPAN_KIND_CLIENT",
+ "startTimeUnixNano": "1795164306281855668",
+ "endTimeUnixNano": "1795164306285749976",
+ "attributes": [
+ {
+ "key": "db.system",
+ "value": {
+ "stringValue": "postgresql"
+ }
+ }
+ ],
+ "status": {}
+ },
+ {
+ "traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
+ "spanId": "bWCElBq0RoM=",
+ "parentSpanId": "ASNFZ4mrze8=",
+ "flags": 768,
+ "name": "POST /api/v1/chats/{chat_id}/attachments",
+ "kind": "SPAN_KIND_SERVER",
+ "startTimeUnixNano": "1795164306279505300",
+ "endTimeUnixNano": "1795164306285992974",
+ "attributes": [
+ {
+ "key": "http.request.method",
+ "value": {
+ "stringValue": "POST"
+ }
+ },
+ {
+ "key": "http.route",
+ "value": {
+ "stringValue": "/api/v1/chats/{chat_id}/attachments"
+ }
+ },
+ {
+ "key": "http.response.status_code",
+ "value": {
+ "intValue": "202"
+ }
+ }
+ ],
+ "status": {}
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "logs": [
+ {
+ "timestamp": "2026-11-20T08:45:06.285760+00:00",
+ "severity": "INFO",
+ "service.name": "messenger-api",
+ "event": "attachment_accepted",
+ "sampled": true,
+ "trace_id": "a630912396b64101b293cc306ee44ed8",
+ "span_id": "6d6084941ab44683",
+ "attachment_id": "a6e6251c-5774-4a3b-8424-c72d9568e4a6",
+ "job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2"
+ },
+ {
+ "timestamp": "2026-11-20T08:45:06.525174+00:00",
+ "severity": "INFO",
+ "service.name": "messenger-worker",
+ "event": "job_processed",
+ "sampled": true,
+ "trace_id": "a630912396b64101b293cc306ee44ed8",
+ "span_id": "c14652912ef9455b",
+ "job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2",
+ "attempt": 1,
+ "outcome": "ready"
+ },
+ {
+ "timestamp": "2026-11-20T08:45:06.353780+00:00",
+ "severity": "INFO",
+ "service.name": "messenger-dispatcher",
+ "event": "job_published",
+ "sampled": true,
+ "trace_id": "a630912396b64101b293cc306ee44ed8",
+ "span_id": "d57e3b5951a6c150",
+ "job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2",
+ "attempt": 1
+ },
+ {
+ "timestamp": "2026-11-20T08:45:06.285952+00:00",
+ "severity": "INFO",
+ "service.name": "messenger-api",
+ "event": "http_request",
+ "sampled": true,
+ "trace_id": "a630912396b64101b293cc306ee44ed8",
+ "span_id": "6d6084941ab44683",
+ "request_id": "cb06618bb089444e8958f45beb2a8d1f",
+ "method": "POST",
+ "route": "/api/v1/chats/{chat_id}/attachments",
+ "status": 202,
+ "duration_seconds": 0.006511
+ },
+ {
+ "timestamp": "2026-11-20T08:45:06.504597+00:00",
+ "severity": "INFO",
+ "service.name": "messenger-worker",
+ "event": "operation_failed",
+ "sampled": true,
+ "trace_id": "a630912396b64101b293cc306ee44ed8",
+ "span_id": "52a483e5ad479b43",
+ "error_type": "ClientError"
+ },
+ {
+ "timestamp": "2026-11-20T08:45:06.510258+00:00",
+ "severity": "INFO",
+ "service.name": "messenger-worker",
+ "event": "operation_failed",
+ "sampled": true,
+ "trace_id": "a630912396b64101b293cc306ee44ed8",
+ "span_id": "b0699f5af6610211",
+ "error_type": "ClientError"
+ },
+ {
+ "timestamp": "2026-11-20T08:45:06.513218+00:00",
+ "severity": "INFO",
+ "service.name": "messenger-worker",
+ "event": "operation_failed",
+ "sampled": true,
+ "trace_id": "a630912396b64101b293cc306ee44ed8",
+ "span_id": "9fbfb420881e78ea",
+ "error_type": "ClientError"
+ },
+ {
+ "timestamp": "2026-11-20T08:45:06.515824+00:00",
+ "severity": "INFO",
+ "service.name": "messenger-worker",
+ "event": "operation_failed",
+ "sampled": true,
+ "trace_id": "a630912396b64101b293cc306ee44ed8",
+ "span_id": "4e1424af20f9e2d5",
+ "error_type": "ClientError"
+ }
+ ]
+}
diff --git a/evidence/trace-ui.jpg b/evidence/trace-ui.jpg
new file mode 100644
index 0000000..b5ff3eb
Binary files /dev/null and b/evidence/trace-ui.jpg differ
diff --git a/infra/RUNBOOK.md b/infra/RUNBOOK.md
new file mode 100644
index 0000000..e5bd1e6
--- /dev/null
+++ b/infra/RUNBOOK.md
@@ -0,0 +1,67 @@
+# Локальный runbook — lab6
+
+Grafana: http://127.0.0.1:13006/d/messenger-lab6.
+Prometheus: http://127.0.0.1:19096.
+Loki и Tempo доступны через Explore и provisioned datasources.
+Не запрашивать credentials, HTTP bodies, raw URL или SQL parameters для диагностики.
+
+## api-errors
+
+1. Открыть API throughput, 5xx ratio и p95/p99. Health исключён.
+2. Проверить APIHighErrorRate в Prometheus /alerts: 5xx / все обработанные business requests за 30s > 5%, for 6s.
+3. В Loki: {service_name="messenger-api"} | json | status >= 500.
+4. Взять trace_id из лога и перейти в Tempo. X-Request-Id связывает ответ с тем же логом.
+5. Если основное время/ошибка в db.transaction, проверить db_pool_used/waiting и QueueObservationUnavailable:
+ проблема на границе PostgreSQL/пула, а не декодирования изображения.
+6. Контролируемый инцидент теста удерживает ACCESS EXCLUSIVE lock таблицы sessions: lock_timeout=1.5s даёт 503.
+ SELECT 1 readiness остаётся доступным; долгий span db.transaction показывает причину, даже если HTTP health зелёный.
+7. После снятия блокировки тот же Bearer снова работает. Подождать очистки 30s окна и for; проверить resolved.
+
+## queue-lag
+
+1. Проверить queue_depth и queue_oldest_age_seconds: это принятые незавершённые jobs в PostgreSQL,
+ включая ещё не опубликованный outbox; это не длина Redis stream и не длительность resize.
+2. Если upload остаётся 202, а возраст/глубина растут, проверить worker busy/capacity и отсутствие новых job_processed.
+ Удалённый процесс может оставаться в scrape cache до 30s; учитывать эту задержку.
+3. В Loki: {service_name=~"messenger-.*"} | json | event=~"attachment_accepted|job_published|job_processed|job_retry".
+4. Через trace_id пройти API → producer → consumer → S3/DB.
+ Published без consumer указывает на отсутствие/недоступность worker или его чтения broker.
+ Consumer с retry и ошибочным S3 span указывает на storage, а не потерю задания.
+5. Восстановить worker. Проверить ready, уменьшение backlog до нуля, исчезновение QueueLag.
+6. Не удалять volume, stream, job registry или staging ради зелёного dashboard.
+
+## telemetry-backend
+
+1. Если business API продолжает отвечать, но trace не находится, проверить Collector:
+ otelcol_exporter_send_failed_spans_total, otelcol_exporter_queue_size/capacity.
+2. Уточнить, принимает ли Collector OTLP (otelcol_receiver_accepted_spans_total).
+ Ошибки exporter при растущем accepted означают проблему downstream, а не отсутствие инструментации.
+3. Восстановить Tempo/Loki; проверить новый trace/log. Старые telemetry, отброшенные после 10s retry budget,
+ не обязаны появиться. Бизнес-данные и telemetry имеют разные гарантии сохранности.
+4. Collector: memory limiter 192MiB, batch <=256, две exporter-очереди по 128 batches;
+ SDK: span queue512/batch128, logs256/batch64, OTLP timeout1s.
+ Нет бесконечной очереди и обязательной блокирующей синхронной отправки на HTTP-path.
+5. У каждого backend именованный volume. Отказ процесса и потеря диска — разные инциденты.
+
+## SLI / SLO
+
+Учебное окно 5 минут, при наличии запросов:
+- API availability SLI = 1 - sum(increase(http_server_requests_total{status_class="5xx"}[5m]))
+ / sum(increase(http_server_requests_total[5m])). Цель >=99%.
+ 4xx считаются корректно обслуженными запросами, health исключён.
+ Это SLI обработанных API запросов: отказ LB/сети до API в знаменатель не попадает.
+- Time-to-ready latency SLI среди завершённых ready jobs:
+ sum(increase(job_ready_duration_seconds_bucket{le="5"}[5m]))
+ / sum(increase(job_ready_duration_seconds_count[5m])). Цель >=95%.
+ Не завершённые/failed jobs не скрываются: отдельно jobs_processed outcomes, job_retries и QueueLag.
+ Этот latency SLO не является SLO доступности всего pipeline.
+
+Короткие алерты 30s и 10s queue age предназначены для локальной демонстрации.
+Они не заявлены промышленными burn-rate alerts; выбран трек 1.
+
+
+Balanced default: TRACE_SAMPLE_RATIO=0.25; metrics/logs всегда включены.
+У несэмплированного запроса trace_id присутствует в log, но полного trace в Tempo не будет.
+Для контролируемого proof используется W3C sampled=1; для полной локальной диагностики:
+TRACE_SAMPLE_RATIO=1 docker compose up -d --no-deps api worker dispatcher.
+Это не tail sampling: ошибки и медленные spans не получают автоматического приоритета.
diff --git a/infra/alerts.yaml b/infra/alerts.yaml
new file mode 100644
index 0000000..39a6f1f
--- /dev/null
+++ b/infra/alerts.yaml
@@ -0,0 +1,27 @@
+groups:
+ - name: messenger
+ rules:
+ - alert: APIHighErrorRate
+ expr: sum(rate(http_server_requests_total{service_name="messenger-api",status_class="5xx"}[30s])) / clamp_min(sum(rate(http_server_requests_total{service_name="messenger-api"}[30s])), 0.001) > 0.05
+ for: 6s
+ labels:
+ severity: warning
+ annotations:
+ summary: API 5xx share exceeds 5 percent in the last 30 seconds
+ runbook: infra/RUNBOOK.md#api-errors
+ - alert: QueueLag
+ expr: max(queue_oldest_age_seconds{service_name="messenger-dispatcher"}) > 10 and max(queue_depth{service_name="messenger-dispatcher"}) > 0
+ for: 6s
+ labels:
+ severity: warning
+ annotations:
+ summary: Accepted unfinished image work older than 10 seconds
+ runbook: infra/RUNBOOK.md#queue-lag
+ - alert: QueueObservationUnavailable
+ expr: min(queue_observation_ok{service_name="messenger-dispatcher"}) == 0
+ for: 6s
+ labels:
+ severity: warning
+ annotations:
+ summary: Dispatcher cannot observe PostgreSQL backlog
+ runbook: infra/RUNBOOK.md#api-errors
diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json
new file mode 100644
index 0000000..f57b7c7
--- /dev/null
+++ b/infra/grafana/dashboards/messenger.json
@@ -0,0 +1,713 @@
+{
+ "uid": "messenger-lab6",
+ "title": "MTUSI lab6 — Messenger observability",
+ "schemaVersion": 41,
+ "version": 1,
+ "refresh": "5s",
+ "time": {
+ "from": "now-15m",
+ "to": "now"
+ },
+ "tags": [
+ "mtusi",
+ "lab6"
+ ],
+ "annotations": {
+ "list": []
+ },
+ "panels": [
+ {
+ "id": 1,
+ "title": "API throughput (health excluded)",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 0,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum(rate(http_server_requests_total{service_name=\"messenger-api\"}[1m]))",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 2,
+ "title": "API 5xx ratio",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 0,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum(rate(http_server_requests_total{service_name=\"messenger-api\",status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{service_name=\"messenger-api\"}[1m])),0.001)",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 3,
+ "title": "API p50 from histogram",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 8,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 4,
+ "title": "API p95 from histogram",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 8,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 5,
+ "title": "API p99 from histogram",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 16,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 6,
+ "title": "Worker jobs by outcome",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 16,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum by (outcome) (rate(jobs_processed_total{service_name=\"messenger-worker\"}[1m]))",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 7,
+ "title": "Worker processing p95",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 24,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 8,
+ "title": "Accepted to ready p95",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 24,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "histogram_quantile(0.95,sum by(le)(rate(job_ready_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 9,
+ "title": "Accepted unfinished jobs",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 32,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "max(queue_depth{service_name=\"messenger-dispatcher\"})",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 10,
+ "title": "Oldest unfinished job age",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 32,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "max(queue_oldest_age_seconds{service_name=\"messenger-dispatcher\"})",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 11,
+ "title": "DB pool used",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 40,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum by(service_name) (db_pool_used{service_name=\"messenger-api\"})",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 12,
+ "title": "DB pool waiting",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 40,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum by(service_name) (db_pool_waiting{service_name=\"messenger-api\"})",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 13,
+ "title": "Worker busy / capacity",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 48,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 14,
+ "title": "Process resident memory",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 48,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "process_resident_memory_bytes{service_name=~\"messenger-.*\"}",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 15,
+ "title": "Process CPU seconds per second",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 56,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum by(service_name)(rate(process_cpu_seconds_total{service_name=~\"messenger-.*\"}[1m]))",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 16,
+ "title": "Container memory incl renderer",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 56,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "container_memory_bytes{service_name=~\"messenger-.*\"}",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 17,
+ "title": "Local alerts",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 64,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "ALERTS{alertstate=\"firing\"}",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 18,
+ "title": "Collector failed span exports",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 64,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))",
+ "legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
+ }
+ ],
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 100,
+ "title": "Correlated structured logs: open TraceID for API / publish / worker chain",
+ "type": "logs",
+ "datasource": {
+ "type": "loki",
+ "uid": "loki"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 72,
+ "w": 24,
+ "h": 10
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "{service_name=~\"messenger-.*\"} | json",
+ "queryType": "range"
+ }
+ ],
+ "options": {
+ "showTime": true,
+ "showLabels": false,
+ "sortOrder": "Descending"
+ }
+ },
+ {
+ "id": 101,
+ "title": "Terminal failed jobs",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 82,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum(rate(job_retries_total{service_name=\"messenger-worker\",outcome=\"failed\"}[1m]))"
+ }
+ ]
+ },
+ {
+ "id": 102,
+ "title": "Retry attempts",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 82,
+ "w": 12,
+ "h": 8
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum(rate(job_retries_total{service_name=\"messenger-worker\",outcome=\"retry\"}[1m]))"
+ }
+ ]
+ }
+ ]
+}
diff --git a/infra/grafana/provisioning/dashboards/provider.yaml b/infra/grafana/provisioning/dashboards/provider.yaml
new file mode 100644
index 0000000..f24ab58
--- /dev/null
+++ b/infra/grafana/provisioning/dashboards/provider.yaml
@@ -0,0 +1,8 @@
+apiVersion: 1
+providers:
+ - name: Messenger
+ folder: MTUSI
+ type: file
+ updateIntervalSeconds: 10
+ options:
+ path: /var/lib/grafana/dashboards
diff --git a/infra/grafana/provisioning/datasources/sources.yaml b/infra/grafana/provisioning/datasources/sources.yaml
new file mode 100644
index 0000000..1c00fbd
--- /dev/null
+++ b/infra/grafana/provisioning/datasources/sources.yaml
@@ -0,0 +1,32 @@
+apiVersion: 1
+datasources:
+ - name: Prometheus
+ uid: prometheus
+ type: prometheus
+ access: proxy
+ url: http://prometheus:9090
+ isDefault: true
+ - name: Loki
+ uid: loki
+ type: loki
+ access: proxy
+ url: http://loki:3100
+ jsonData:
+ derivedFields:
+ - name: TraceID
+ datasourceUid: tempo
+ matcherRegex: '"trace_id":"([a-f0-9]{32})"'
+ url: '$${__value.raw}'
+ - name: Tempo
+ uid: tempo
+ type: tempo
+ access: proxy
+ url: http://tempo:3200
+ jsonData:
+ tracesToLogsV2:
+ datasourceUid: loki
+ spanStartTimeShift: '-1m'
+ spanEndTimeShift: '1m'
+ filterByTraceID: false
+ customQuery: true
+ query: '{service_name=~"messenger-.*"} | json | trace_id="$${__trace.traceId}"'
diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg
new file mode 100644
index 0000000..5495e1e
--- /dev/null
+++ b/infra/haproxy.cfg
@@ -0,0 +1,46 @@
+global
+ maxconn 1024
+ nbthread 1
+ hard-stop-after 10s
+
+defaults
+ mode http
+ timeout connect 500ms
+ timeout check 500ms
+ timeout queue 1s
+ timeout client 30s
+ timeout server 30s
+ timeout http-request 3s
+ retries 2
+ retry-on conn-failure
+ option http-server-close
+
+resolvers docker
+ nameserver docker_dns 127.0.0.11:53
+ accepted_payload_size 8192
+ resolve_retries 2
+ timeout resolve 1s
+ timeout retry 1s
+ hold valid 1s
+ hold nx 1s
+ hold refused 1s
+ hold timeout 1s
+ hold other 1s
+ hold obsolete 1s
+
+frontend public
+ bind :8080
+ default_backend replicas
+
+backend replicas
+ balance roundrobin
+ option httpchk
+ http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
+ http-check expect status 200
+ default-server inter 1s fall 2 rise 2
+ server-template api 1-4 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
+
+listen metrics
+ bind :8404
+ stats enable
+ stats uri /stats
diff --git a/infra/loki.yaml b/infra/loki.yaml
new file mode 100644
index 0000000..244cca6
--- /dev/null
+++ b/infra/loki.yaml
@@ -0,0 +1,33 @@
+auth_enabled: false
+server:
+ http_listen_port: 3100
+common:
+ path_prefix: /loki
+ replication_factor: 1
+ ring:
+ kvstore:
+ store: inmemory
+ storage:
+ filesystem:
+ chunks_directory: /loki/chunks
+ rules_directory: /loki/rules
+schema_config:
+ configs:
+ - from: 2024-01-01
+ store: tsdb
+ object_store: filesystem
+ schema: v13
+ index:
+ prefix: index_
+ period: 24h
+limits_config:
+ allow_structured_metadata: true
+ retention_period: 24h
+ ingestion_rate_mb: 4
+ ingestion_burst_size_mb: 8
+compactor:
+ working_directory: /loki/compactor
+ retention_enabled: true
+ delete_request_store: filesystem
+analytics:
+ reporting_enabled: false
diff --git a/infra/otel-collector.yaml b/infra/otel-collector.yaml
new file mode 100644
index 0000000..eea2b42
--- /dev/null
+++ b/infra/otel-collector.yaml
@@ -0,0 +1,66 @@
+receivers:
+ otlp:
+ protocols:
+ http:
+ endpoint: 0.0.0.0:4318
+processors:
+ memory_limiter:
+ check_interval: 1s
+ limit_mib: 192
+ spike_limit_mib: 32
+ batch:
+ timeout: 1s
+ send_batch_size: 128
+ send_batch_max_size: 256
+exporters:
+ prometheus:
+ endpoint: 0.0.0.0:8889
+ metric_expiration: 30s
+ resource_to_telemetry_conversion:
+ enabled: true
+ otlphttp/tempo:
+ endpoint: http://tempo:4318
+ timeout: 2s
+ sending_queue:
+ enabled: true
+ num_consumers: 2
+ queue_size: 128
+ retry_on_failure:
+ enabled: true
+ initial_interval: 1s
+ max_interval: 3s
+ max_elapsed_time: 10s
+ otlphttp/loki:
+ endpoint: http://loki:3100/otlp
+ timeout: 2s
+ sending_queue:
+ enabled: true
+ num_consumers: 2
+ queue_size: 128
+ retry_on_failure:
+ enabled: true
+ initial_interval: 1s
+ max_interval: 3s
+ max_elapsed_time: 10s
+service:
+ telemetry:
+ metrics:
+ readers:
+ - pull:
+ exporter:
+ prometheus:
+ host: 0.0.0.0
+ port: 8888
+ pipelines:
+ traces:
+ receivers: [otlp]
+ processors: [memory_limiter, batch]
+ exporters: [otlphttp/tempo]
+ metrics:
+ receivers: [otlp]
+ processors: [memory_limiter, batch]
+ exporters: [prometheus]
+ logs:
+ receivers: [otlp]
+ processors: [memory_limiter, batch]
+ exporters: [otlphttp/loki]
diff --git a/infra/prometheus.yaml b/infra/prometheus.yaml
new file mode 100644
index 0000000..e2f4673
--- /dev/null
+++ b/infra/prometheus.yaml
@@ -0,0 +1,11 @@
+global:
+ scrape_interval: 2s
+ evaluation_interval: 2s
+rule_files: [/etc/prometheus/alerts.yaml]
+scrape_configs:
+ - job_name: messenger
+ static_configs:
+ - targets: [otel-collector:8889]
+ - job_name: collector
+ static_configs:
+ - targets: [otel-collector:8888]
diff --git a/infra/tempo.yaml b/infra/tempo.yaml
new file mode 100644
index 0000000..1392b9b
--- /dev/null
+++ b/infra/tempo.yaml
@@ -0,0 +1,24 @@
+server:
+ http_listen_port: 3200
+distributor:
+ receivers:
+ otlp:
+ protocols:
+ http:
+ endpoint: 0.0.0.0:4318
+ingester:
+ trace_idle_period: 5s
+ max_block_duration: 1m
+ max_block_bytes: 1048576
+compactor:
+ compaction:
+ block_retention: 24h
+storage:
+ trace:
+ backend: local
+ wal:
+ path: /var/tempo/wal
+ local:
+ path: /var/tempo/blocks
+usage_report:
+ reporting_enabled: false
diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql
new file mode 100644
index 0000000..eeac3c1
--- /dev/null
+++ b/migrations/001_initial.sql
@@ -0,0 +1,42 @@
+CREATE TABLE users (
+ id UUID PRIMARY KEY
+ , username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$')
+ , display_name TEXT NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100)
+ , password_hash TEXT NOT NULL
+ , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+)
+;
+CREATE TABLE chats (
+ id UUID PRIMARY KEY
+ , title TEXT NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100)
+ , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+ , last_message_at TIMESTAMPTZ
+)
+;
+CREATE TABLE chat_members (
+ chat_id UUID NOT NULL REFERENCES chats(id)
+ , user_id UUID NOT NULL REFERENCES users(id)
+ , PRIMARY KEY (chat_id, user_id)
+)
+;
+CREATE INDEX chat_members_user ON chat_members (user_id, chat_id)
+;
+CREATE TABLE messages (
+ id UUID PRIMARY KEY
+ , chat_id UUID NOT NULL REFERENCES chats(id)
+ , sender_id UUID NOT NULL REFERENCES users(id)
+ , text TEXT NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000)
+ , created_at TIMESTAMPTZ NOT NULL
+ , FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
+)
+;
+CREATE INDEX messages_history ON messages (chat_id, created_at, id)
+;
+CREATE TABLE sessions (
+ token_hash TEXT PRIMARY KEY
+ , user_id UUID NOT NULL REFERENCES users(id)
+ , expires_at TIMESTAMPTZ NOT NULL
+)
+;
+CREATE INDEX sessions_expiry ON sessions (expires_at)
+;
diff --git a/migrations/002_unicode_nul.sql b/migrations/002_unicode_nul.sql
new file mode 100644
index 0000000..224cda0
--- /dev/null
+++ b/migrations/002_unicode_nul.sql
@@ -0,0 +1,17 @@
+-- PostgreSQL TEXT cannot contain U+0000. Store validated UTF-8 losslessly in BYTEA.
+-- API validation keeps the contract's code-point limits; these bounds cap bytes.
+ALTER TABLE users
+ DROP CONSTRAINT users_display_name_check
+ , ALTER COLUMN display_name TYPE BYTEA USING convert_to(display_name, 'UTF8')
+ , ADD CONSTRAINT users_display_name_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400)
+;
+ALTER TABLE chats
+ DROP CONSTRAINT chats_title_check
+ , ALTER COLUMN title TYPE BYTEA USING convert_to(title, 'UTF8')
+ , ADD CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400)
+;
+ALTER TABLE messages
+ DROP CONSTRAINT messages_text_check
+ , ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8')
+ , ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000)
+;
diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql
new file mode 100644
index 0000000..3ca53a7
--- /dev/null
+++ b/migrations/003_idempotency.sql
@@ -0,0 +1,12 @@
+CREATE TABLE idempotency_keys (
+ user_id UUID NOT NULL REFERENCES users(id)
+ , chat_id UUID NOT NULL REFERENCES chats(id)
+ , key VARCHAR(128) NOT NULL
+ , request_hash CHAR(64) NOT NULL
+ , message_id UUID REFERENCES messages(id)
+ , expires_at TIMESTAMPTZ NOT NULL
+ , PRIMARY KEY (user_id, chat_id, key)
+)
+;
+CREATE INDEX idempotency_expiry ON idempotency_keys (expires_at)
+;
diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql
new file mode 100644
index 0000000..bfd5189
--- /dev/null
+++ b/migrations/004_attachments.sql
@@ -0,0 +1,30 @@
+ALTER TABLE messages
+ DROP CONSTRAINT messages_text_bytes
+ , ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000)
+ , ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}'
+;
+CREATE TABLE attachments (
+ id UUID PRIMARY KEY
+ , chat_id UUID NOT NULL REFERENCES chats(id)
+ , owner_id UUID NOT NULL REFERENCES users(id)
+ , filename TEXT NOT NULL
+ , object_key TEXT NOT NULL UNIQUE
+ , content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg'))
+ , size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760)
+ , sha256 CHAR(64) NOT NULL
+ , state TEXT NOT NULL CHECK (state IN ('uploading', 'ready', 'storage_missing'))
+ , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+ , updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+ , FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id)
+)
+;
+CREATE INDEX attachments_chat ON attachments (chat_id, id)
+;
+CREATE TABLE message_attachments (
+ message_id UUID NOT NULL REFERENCES messages(id)
+ , attachment_id UUID NOT NULL REFERENCES attachments(id)
+ , PRIMARY KEY (message_id, attachment_id)
+)
+;
+CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id)
+;
diff --git a/migrations/005_pipeline.sql b/migrations/005_pipeline.sql
new file mode 100644
index 0000000..0ef1e81
--- /dev/null
+++ b/migrations/005_pipeline.sql
@@ -0,0 +1,40 @@
+ALTER TABLE attachments
+ DROP CONSTRAINT attachments_state_check
+ , ADD CONSTRAINT attachments_state_check CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed', 'storage_missing'))
+ , ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1
+ , ADD COLUMN staging_key TEXT
+ , ADD COLUMN variants JSONB NOT NULL DEFAULT '[]'
+ , ADD COLUMN error_code TEXT
+ , ADD COLUMN current_job_id UUID
+;
+CREATE TABLE attachment_jobs (
+ id UUID PRIMARY KEY
+ , attachment_id UUID NOT NULL REFERENCES attachments(id)
+ , pipeline_version INTEGER NOT NULL
+ , source_key TEXT NOT NULL
+ , state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded'))
+ , attempt INTEGER NOT NULL DEFAULT 0
+ , max_attempts INTEGER NOT NULL DEFAULT 5
+ , lease_token UUID
+ , lease_until TIMESTAMPTZ
+ , next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+ , published_at TIMESTAMPTZ
+ , error_code TEXT
+ , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+ , updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+ , UNIQUE (attachment_id, pipeline_version)
+)
+;
+ALTER TABLE attachments
+ ADD CONSTRAINT attachments_current_job FOREIGN KEY (current_job_id) REFERENCES attachment_jobs(id)
+;
+CREATE INDEX jobs_due ON attachment_jobs (state, next_attempt_at)
+;
+CREATE TABLE job_events (
+ id BIGSERIAL PRIMARY KEY
+ , job_id UUID NOT NULL REFERENCES attachment_jobs(id)
+ , event TEXT NOT NULL
+ , attempt INTEGER NOT NULL
+ , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+)
+;
diff --git a/migrations/006_observability.sql b/migrations/006_observability.sql
new file mode 100644
index 0000000..e19783d
--- /dev/null
+++ b/migrations/006_observability.sql
@@ -0,0 +1,3 @@
+ALTER TABLE attachment_jobs
+ ADD COLUMN trace_context JSONB NOT NULL DEFAULT '{}'
+;
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..7fa5df3
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,27 @@
+boto3==1.40.61
+botocore==1.40.76
+certifi==2026.10.23
+charset-normalizer==3.5.2
+googleapis-common-protos==1.76.0
+idna==3.20
+importlib_metadata==8.7.1
+jmespath==1.1.0
+opentelemetry-api==1.36.0
+opentelemetry-exporter-otlp-proto-common==1.36.0
+opentelemetry-exporter-otlp-proto-http==1.36.0
+opentelemetry-proto==1.36.0
+opentelemetry-sdk==1.36.0
+opentelemetry-semantic-conventions==0.57b0
+pillow==13.0.0
+protobuf==6.33.6
+psycopg==3.2.3
+psycopg-binary==3.2.3
+psycopg-pool==3.2.4
+python-dateutil==2.9.0.post0
+redis==5.2.1
+requests==2.34.2
+s3transfer==0.14.0
+six==1.17.0
+typing_extensions==4.16.0
+urllib3==2.8.0
+zipp==4.1.2
diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py
new file mode 100644
index 0000000..4b118cb
--- /dev/null
+++ b/scripts/bootstrap.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Create local credentials without printing or overwriting existing values."""
+import json
+from pathlib import Path
+import secrets
+
+root = Path(__file__).resolve().parents[1]
+directory = root / 'secrets'
+directory.mkdir(mode=0o700, exist_ok=True)
+
+def save(name, value):
+ path = directory / name
+ if not path.exists():
+ path.write_text(value)
+ path.chmod(0o644) # UID 10001 in Compose; parent is owner-only 0700.
+ return path.read_text().strip()
+
+password = save('postgres_password', secrets.token_hex(24))
+save('database_url', 'postgresql://messenger:' + password + '@postgres:5432/messenger')
+save('cache_url', 'redis://redis:6379/0')
+save('broker_url', 'redis://broker:6379/0')
+identities = []
+for role, actions in {
+ 'admin': ['Admin', 'Read', 'Write', 'List'],
+ 'api': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'],
+ 'worker': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'],
+}.items():
+ key = save('s3_' + role + '_key', secrets.token_hex(12))
+ secret = save('s3_' + role + '_secret', secrets.token_hex(24))
+ identities.append({'name': role, 'credentials': [{'accessKey': key, 'secretKey': secret}], 'actions': actions})
+configuration = directory / 's3_config'
+configuration.write_text(json.dumps({'identities': identities}))
+configuration.chmod(0o600) # Only the root S3 gateway mounts the full identity configuration.
+env = root / '.env'
+if not env.exists():
+ env.write_text((root / '.env.example').read_text())
+ env.chmod(0o600)
+print('Local configuration ready; no secrets printed.')
diff --git a/scripts/restart.sh b/scripts/restart.sh
new file mode 100755
index 0000000..dfde59d
--- /dev/null
+++ b/scripts/restart.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+python3 scripts/snapshot.py > evidence/restart-before.json
+docker compose stop api lb worker dispatcher
+docker compose restart postgres redis s3-store s3 broker otel-collector prometheus loki tempo grafana
+docker compose up -d --wait postgres redis s3-store s3 broker otel-collector prometheus loki tempo grafana
+docker compose up -d --wait api lb worker dispatcher
+python3 scripts/snapshot.py > evidence/restart-after.json
diff --git a/scripts/snapshot.py b/scripts/snapshot.py
new file mode 100644
index 0000000..3d0354f
--- /dev/null
+++ b/scripts/snapshot.py
@@ -0,0 +1,11 @@
+"""Sanitized process and resource evidence; never dump container environments."""
+import json
+import subprocess
+
+ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split()
+records = []
+for container in ids:
+ raw = subprocess.check_output(['docker', 'inspect', container], text=True)
+ data = json.loads(raw)[0]
+ records.append({'container': data['Id'], 'name': data['Name'], 'service': data['Config']['Labels']['com.docker.compose.service'], 'image': data['Config']['Image'], 'user': data['Config']['User'], 'running': data['State']['Running'], 'started_at': data['State']['StartedAt'], 'exit_code': data['State']['ExitCode'], 'oom_killed': data['State']['OOMKilled'], 'memory_limit': data['HostConfig']['Memory'], 'nano_cpus': data['HostConfig']['NanoCpus'], 'ports': data['NetworkSettings']['Ports'], 'mounts': [{'type': m['Type'], 'destination': m['Destination'], 'name': m.get('Name')} for m in data['Mounts']]})
+print(json.dumps(records, indent=2))
diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh
new file mode 100755
index 0000000..ce9279f
--- /dev/null
+++ b/scripts/stop-one.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+exec python3 scripts/stop_one.py
diff --git a/scripts/stop_one.py b/scripts/stop_one.py
new file mode 100644
index 0000000..bc495aa
--- /dev/null
+++ b/scripts/stop_one.py
@@ -0,0 +1,32 @@
+"""Resolve the requested process alias using each actual container, never a fake header."""
+import json
+import os
+from pathlib import Path
+import subprocess
+import time
+
+
+def docker(*args):
+ return subprocess.check_output(['docker', *args], text=True).strip()
+
+
+def instance(container):
+ code = "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=1).headers['X-Instance-Id'])"
+ return docker('exec', container, 'python', '-c', code)
+
+
+if __name__ == '__main__':
+ target = os.environ['TARGET_INSTANCE_ID']
+ started = time.monotonic()
+ for container in docker('compose', 'ps', '-q', 'api').splitlines():
+ if instance(container) == target:
+ docker('stop', '--time', '8', container)
+ state = json.loads(docker('inspect', '--format', '{{json .State}}', container))
+ assert not state['Running'], 'target must remain stopped'
+ record = {'instance': target, 'container': container, 'running': state['Running'], 'exit_code': state['ExitCode'], 'seconds': round(time.monotonic() - started, 3)}
+ with Path('evidence/stopped-instances.jsonl').open('a') as out:
+ out.write(json.dumps(record) + '\n')
+ print(json.dumps(record))
+ break
+ else:
+ raise SystemExit('Requested live API instance not found')
diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py
new file mode 100644
index 0000000..e635cae
--- /dev/null
+++ b/scripts/wait_ready.py
@@ -0,0 +1,16 @@
+import os
+import time
+import urllib.request
+
+base = os.environ.get('BASE_URL', 'http://127.0.0.1:18087')
+for attempt in range(120):
+ try:
+ with urllib.request.urlopen(base + '/health/ready', timeout=2) as response:
+ if response.status == 200:
+ print('Ready through load balancer.')
+ break
+ except OSError:
+ pass
+ time.sleep(0.5)
+else:
+ raise SystemExit('Readiness timeout')
diff --git a/src/admin.py b/src/admin.py
new file mode 100644
index 0000000..699bd3e
--- /dev/null
+++ b/src/admin.py
@@ -0,0 +1,127 @@
+"""Explicit local operator tools; no administrative HTTP endpoint."""
+import argparse
+import json
+import os
+import uuid
+
+from pipeline import connect, audit
+from objects import Objects
+from postgres import setting
+
+
+def replay(job_id):
+ with connect() as conn, conn.transaction():
+ job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job_id,)).fetchone()
+ if job is None or job['state'] != 'failed':
+ raise ValueError('Replay requires a failed job')
+ attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND current_job_id = %s FOR UPDATE', (job_id,)).fetchone()
+ if attachment is None or attachment['state'] != 'failed' or Objects(setting).head(job['source_key']) is None:
+ raise ValueError('Current failed source is unavailable')
+ conn.execute('''
+ UPDATE attachment_jobs
+ SET state = 'queued'
+ , max_attempts = attempt + %s
+ , published_at = NULL
+ , next_attempt_at = clock_timestamp()
+ , error_code = NULL
+ , updated_at = clock_timestamp()
+ WHERE 1=1
+ AND id = %s
+ ''', (int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), job_id))
+ conn.execute("UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job_id,))
+ audit(conn, job, 'operator_replay')
+ return {'job_id': str(job_id), 'replayed': True}
+
+
+def version(attachment_id, number):
+ if number not in (1, 2):
+ raise ValueError('Only implemented pipeline versions 1 and 2 are supported')
+ with connect() as conn, conn.transaction():
+ row = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone()
+ if row is None or number != row['pipeline_version'] + 1 or row['state'] in ('uploading', 'failed', 'storage_missing'):
+ raise ValueError('Version must advance the current process exactly once')
+ old = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (row['current_job_id'],)).fetchone()
+ source = row['object_key'] if row['state'] == 'ready' else old['source_key']
+ if Objects(setting).head(source) is None:
+ raise ValueError('Source unavailable')
+ conn.execute("UPDATE attachment_jobs SET state = 'superseded', lease_token = NULL, lease_until = NULL WHERE 1=1 AND id = %s", (old['id'],))
+ audit(conn, old, 'operator_supersede')
+ job_id = uuid.uuid4()
+ conn.execute('''
+ INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts)
+ VALUES (%s, %s, %s, %s, %s)
+ ''', (job_id, attachment_id, number, source, int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))))
+ conn.execute('''
+ UPDATE attachments
+ SET pipeline_version = %s
+ , current_job_id = %s
+ , state = 'queued'
+ , variants = '[]'
+ , error_code = NULL
+ , updated_at = clock_timestamp()
+ WHERE 1=1
+ AND id = %s
+ ''', (number, job_id, attachment_id))
+ return {'job_id': str(job_id), 'version': number}
+
+
+def cleanup(apply=False, retention=86400):
+ if retention < 86400:
+ raise ValueError('Retention must be >=24 hours')
+ objects = Objects(setting)
+ counts = {'ready_staging': 0, 'expired_staging': 0}
+ with connect() as conn:
+ # Intent rows cover every valid staging PUT. Never collect active jobs.
+ last = '00000000-0000-0000-0000-000000000000'
+ while True:
+ rows = conn.execute('SELECT id FROM attachments WHERE 1=1 AND id > %s ORDER BY id LIMIT 100', (last,)).fetchall()
+ if not rows:
+ break
+ for candidate in rows:
+ last = candidate['id']
+ with conn.transaction():
+ row = conn.execute('''
+ SELECT *
+ FROM attachments
+ WHERE 1=1
+ AND id = %s
+ AND (state = 'ready' OR (state IN ('uploading', 'failed') AND updated_at < clock_timestamp() - %s * INTERVAL '1 second'))
+ FOR UPDATE SKIP LOCKED
+ ''', (last, retention)).fetchone()
+ if row is None or not row['staging_key'] or not row['staging_key'].startswith('staging/'):
+ continue
+ counts['ready_staging' if row['state'] == 'ready' else 'expired_staging'] += 1
+ if apply:
+ objects.delete(row['staging_key'])
+ if row['state'] == 'uploading' and row['current_job_id'] is None:
+ conn.execute('DELETE FROM attachments WHERE 1=1 AND id = %s', (last,))
+ else:
+ conn.execute('UPDATE attachments SET staging_key = NULL WHERE 1=1 AND id = %s', (last,))
+ return dict(apply=apply, retention_seconds=retention, counts=counts)
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument('action', choices=['failed', 'replay', 'version', 'cleanup'])
+ p.add_argument('identifier', nargs='?')
+ p.add_argument('--version', type=int)
+ p.add_argument('--apply', action='store_true')
+ args = p.parse_args()
+ try:
+ if args.action == 'failed':
+ with connect() as conn:
+ result = conn.execute("SELECT id, attachment_id, pipeline_version, attempt, max_attempts, error_code FROM attachment_jobs WHERE 1=1 AND state = 'failed' ORDER BY created_at LIMIT 100").fetchall()
+ elif args.action == 'replay':
+ result = replay(uuid.UUID(args.identifier))
+ elif args.action == 'version':
+ result = version(uuid.UUID(args.identifier), args.version)
+ else:
+ result = cleanup(args.apply)
+ print(json.dumps(result, default=str, indent=2))
+ except Exception:
+ print(json.dumps({'error': 'Operator action refused or dependency unavailable; no secrets logged.'}))
+ raise SystemExit(1)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/bucket_bootstrap.py b/src/bucket_bootstrap.py
new file mode 100644
index 0000000..a0a2cbd
--- /dev/null
+++ b/src/bucket_bootstrap.py
@@ -0,0 +1,13 @@
+"""Idempotent private bucket bootstrap, executed only with administrative S3 keys."""
+from botocore.exceptions import ClientError
+from objects import Objects
+from postgres import setting
+
+objects = Objects(setting)
+try:
+ objects.client.create_bucket(Bucket=objects.bucket)
+except ClientError as exc:
+ if exc.response['Error']['Code'] not in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
+ raise RuntimeError('Bucket bootstrap failed') from None
+objects.ready()
+print('Private bucket ready; static authenticated identities loaded from local secret config.')
diff --git a/src/common.py b/src/common.py
new file mode 100644
index 0000000..29c87c5
--- /dev/null
+++ b/src/common.py
@@ -0,0 +1,98 @@
+"""Shared HTTP contract; storage is supplied by each laboratory."""
+import base64
+import datetime as dt
+import json
+import re
+import uuid
+
+
+class APIError(Exception):
+ def __init__(self, status, code, message):
+ self.status, self.code, self.message = status, code, message
+
+
+def invalid():
+ raise APIError(400, 'invalid_request', 'Неверные параметры запроса')
+
+
+def unauthorized():
+ raise APIError(401, 'unauthorized', 'Требуется действительная идентификация')
+
+
+def missing():
+ raise APIError(404, 'not_found', 'Объект не найден')
+
+
+def identifier(value):
+ try:
+ if not isinstance(value, str):
+ invalid()
+ return str(uuid.UUID(value))
+ except ValueError:
+ invalid()
+
+
+def text(value, maximum, minimum=1):
+ if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
+ invalid()
+ try:
+ value.encode('utf-8')
+ except UnicodeError:
+ invalid()
+ return value
+
+
+def fields(value, required):
+ if not isinstance(value, dict) or set(value) != set(required):
+ invalid()
+
+
+def username(value):
+ if not isinstance(value, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', value):
+ invalid()
+ return value
+
+
+def now():
+ return dt.datetime.now(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
+
+
+def page_args(query, scope):
+ if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()):
+ invalid()
+ limit = query.get('limit', ['50'])[0]
+ if not re.fullmatch(r'[0-9]+', limit):
+ invalid()
+ try:
+ limit = int(limit)
+ except ValueError:
+ invalid()
+ if not 1 <= limit <= 100:
+ invalid()
+ cursor = None
+ if 'cursor' in query:
+ raw = query['cursor'][0]
+ try:
+ if not raw or len(raw) > 1024:
+ invalid()
+ decoded = json.loads(base64.b64decode(raw + '=' * (-len(raw) % 4), altchars=b'-_', validate=True))
+ if not isinstance(decoded, list) or len(decoded) != 3 or decoded[0] != scope:
+ invalid()
+ stamp = decoded[1]
+ if not isinstance(stamp, str) or not stamp.endswith('Z'):
+ invalid()
+ stamp = dt.datetime.fromisoformat(stamp[:-1] + '+00:00').isoformat(timespec='microseconds').replace('+00:00', 'Z')
+ cursor = (stamp, identifier(decoded[2]))
+ except (ValueError, TypeError, UnicodeError):
+ invalid()
+ return limit, cursor
+
+
+def page(items, limit, scope):
+ more = len(items) > limit
+ items = items[:limit]
+ cursor = None
+ if more:
+ last = items[-1]
+ cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=')
+ return {'items': items, 'next_cursor': cursor}
diff --git a/src/migrate.py b/src/migrate.py
new file mode 100644
index 0000000..ebd85f7
--- /dev/null
+++ b/src/migrate.py
@@ -0,0 +1,13 @@
+"""One-shot migration job; replicas never execute DDL."""
+from pathlib import Path
+import psycopg
+from postgres import setting
+
+with psycopg.connect(setting('DATABASE_URL'), connect_timeout=5) as conn:
+ conn.execute('SELECT pg_advisory_xact_lock(2301003)')
+ conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)')
+ for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
+ if not conn.execute('SELECT version FROM schema_migrations WHERE 1=1 AND version = %s', (path.name,)).fetchone():
+ conn.execute(path.read_text())
+ conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.name,))
+ print('Applied', path.name)
diff --git a/src/objects.py b/src/objects.py
new file mode 100644
index 0000000..b4c5749
--- /dev/null
+++ b/src/objects.py
@@ -0,0 +1,158 @@
+"""Bounded S3 operations. No root credentials or filename-derived object keys."""
+import contextlib
+import telemetry as tele
+import hashlib
+import os
+import tempfile
+import threading
+import time
+import unicodedata
+import warnings
+
+import boto3
+from botocore.config import Config
+from botocore.exceptions import BotoCoreError, ClientError
+from PIL import Image, PngImagePlugin, UnidentifiedImageError
+
+from common import APIError, invalid
+
+MAX_BYTES = 10 * 1024 * 1024
+MAX_PIXELS = 40_000_000
+Image.MAX_IMAGE_PIXELS = MAX_PIXELS
+PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024
+PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
+warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
+UPLOAD_SLOTS = threading.BoundedSemaphore(8)
+DECODE_SLOTS = threading.BoundedSemaphore(1)
+
+
+def unavailable():
+ raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно')
+
+
+def filename(value):
+ if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip():
+ invalid()
+ if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value):
+ invalid()
+ try:
+ value.encode('utf-8')
+ except UnicodeError:
+ invalid()
+ return value
+
+
+@contextlib.contextmanager
+def bounded_upload():
+ if not UPLOAD_SLOTS.acquire(blocking=False):
+ raise APIError(503, 'unavailable', 'Все слоты загрузки заняты')
+ try:
+ yield
+ finally:
+ UPLOAD_SLOTS.release()
+
+
+@contextlib.contextmanager
+def receive(handler):
+ lengths = handler.headers.get_all('Content-Length', [])
+ if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'):
+ invalid()
+ try:
+ length = int(lengths[0])
+ except ValueError:
+ invalid()
+ if length > MAX_BYTES:
+ raise APIError(413, 'payload_too_large', 'Максимум 10 MiB')
+ if length <= 0:
+ invalid()
+ mime = handler.headers.get_content_type()
+ if mime not in ('image/png', 'image/jpeg'):
+ raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG')
+ deadline = time.monotonic() + 15
+ digest = hashlib.sha256()
+ # Ephemeral bounded spool is never a durable source of a confirmed attachment.
+ with tempfile.TemporaryFile(mode='w+b') as source:
+ remaining = length
+ while remaining:
+ handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic())))
+ if time.monotonic() >= deadline:
+ invalid()
+ try:
+ chunk = handler.rfile.read1(min(64 * 1024, remaining))
+ except (OSError, TimeoutError):
+ invalid()
+ if not chunk:
+ invalid()
+ digest.update(chunk)
+ source.write(chunk)
+ remaining -= len(chunk)
+ if not DECODE_SLOTS.acquire(timeout=10):
+ raise APIError(503, 'unavailable', 'Проверка изображений перегружена')
+ try:
+ source.flush()
+ for decode in (False,):
+ source.seek(0)
+ # Pillow.close releases pixel buffers and may close its input file.
+ # A duplicate descriptor preserves the original for the S3 PUT.
+ with os.fdopen(os.dup(source.fileno()), 'rb') as image_file:
+ image = Image.open(image_file, formats=['PNG', 'JPEG'])
+ try:
+ if image.width * image.height > MAX_PIXELS:
+ raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей')
+ if Image.MIME[image.format] != mime:
+ raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению')
+ if decode:
+ image.load()
+ else:
+ image.verify()
+ finally:
+ image.close() # Free pixels BEFORE releasing the decoder semaphore.
+ except (Image.DecompressionBombWarning, Image.DecompressionBombError):
+ raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None
+ except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
+ raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None
+ finally:
+ DECODE_SLOTS.release()
+ source.seek(0)
+ yield source, length, mime, digest.hexdigest()
+
+
+class Objects:
+ def __init__(self, setting):
+ self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments')
+ self.client = boto3.client('s3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:8333'), region_name='us-east-1', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required'))
+ self.client = tele.S3Client(self.client)
+
+ def head(self, key):
+ try:
+ return self.client.head_object(Bucket=self.bucket, Key=key)
+ except ClientError as exc:
+ if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'):
+ return None
+ unavailable()
+ except BotoCoreError:
+ unavailable()
+
+ def ready(self):
+ try:
+ self.client.head_bucket(Bucket=self.bucket)
+ except (ClientError, BotoCoreError):
+ unavailable()
+
+ def put(self, key, source, length, mime, digest):
+ try:
+ self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest})
+ except (ClientError, BotoCoreError):
+ unavailable()
+
+ def get(self, key):
+ try:
+ return self.client.get_object(Bucket=self.bucket, Key=key)
+ except (ClientError, BotoCoreError):
+ unavailable()
+
+ def delete(self, key):
+ try:
+ self.client.delete_object(Bucket=self.bucket, Key=key)
+ except (ClientError, BotoCoreError):
+ unavailable()
diff --git a/src/pipeline.py b/src/pipeline.py
new file mode 100644
index 0000000..2627a60
--- /dev/null
+++ b/src/pipeline.py
@@ -0,0 +1,393 @@
+"""Durable outbox, Redis Streams delivery, bounded worker and fenced result commits."""
+import argparse
+import concurrent.futures
+import contextlib
+import datetime as dt
+import hashlib
+import json
+import os
+from pathlib import Path
+import random
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import uuid
+import telemetry as tele
+
+import psycopg
+from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
+import redis
+from botocore.exceptions import ClientError, BotoCoreError
+from common import APIError
+from objects import Objects, MAX_BYTES
+from postgres import setting
+
+STREAM, GROUP = 'attachment-jobs', 'image-workers'
+LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '60'))
+STOP = threading.Event()
+
+
+def raw_connect():
+ return psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row, autocommit=True, connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1500')
+
+
+@contextlib.contextmanager
+def connect():
+ with tele.span('db.transaction', {'db.system':'postgresql'}, tele.SpanKind.CLIENT):
+ with raw_connect() as connection:
+ yield connection
+
+
+def broker():
+ client = redis.Redis.from_url(setting('BROKER_URL'), decode_responses=True, socket_connect_timeout=1, socket_timeout=3)
+ try:
+ client.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
+ except redis.ResponseError as error:
+ if 'BUSYGROUP' not in str(error):
+ raise
+ return client
+
+
+def event(job):
+ return {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], 'attempt': job['attempt'] + 1, 'created_at': job['created_at'].isoformat().replace('+00:00', 'Z'), **(job.get('trace_context') or {})}
+
+
+def validate(message):
+ required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
+ if not isinstance(message, dict) or set(message) - required - {'traceparent', 'tracestate'} or not required <= set(message):
+ return False
+ try:
+ uuid.UUID(message['job_id']); uuid.UUID(message['attachment_id'])
+ stamp = dt.datetime.fromisoformat(message['created_at'].replace('Z', '+00:00'))
+ if stamp.tzinfo is None or not message['created_at'].endswith('Z'):
+ return False
+ if 'traceparent' in message and (not isinstance(message['traceparent'], str) or not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', message['traceparent'])):
+ return False
+ if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512):
+ return False
+ return type(message['schema_version']) is int and message['schema_version'] == 1 and message['type'] == 'attachment.process.v1' and all(type(message[k]) is int and message[k] >= 1 for k in ['pipeline_version', 'attempt'])
+ except (ValueError, TypeError, AttributeError):
+ return False
+
+
+def audit(conn, job, name):
+ conn.execute('INSERT INTO job_events (job_id, event, attempt) VALUES (%s, %s, %s)', (job['id'], name, job['attempt']))
+
+
+def retry(conn, job, code, permanent=False):
+ failed = permanent or job['attempt'] >= job['max_attempts']
+ state = 'failed' if failed else 'queued'
+ code = code if permanent or not failed else 'retry_exhausted'
+ delay = min(8, .5 * 2 ** min(job['attempt'] - 1, 4)) + random.uniform(0, .5)
+ conn.execute('''
+ UPDATE attachment_jobs
+ SET state = %s
+ , error_code = %s
+ , lease_token = NULL
+ , lease_until = NULL
+ , published_at = NULL
+ , next_attempt_at = clock_timestamp() + %s * INTERVAL '1 second'
+ , updated_at = clock_timestamp()
+ WHERE 1=1
+ AND id = %s
+ ''', (state, code, delay, job['id']))
+ conn.execute('''
+ UPDATE attachments
+ SET state = %s
+ , error_code = %s
+ , updated_at = clock_timestamp()
+ WHERE 1=1
+ AND current_job_id = %s
+ ''', (state, code if failed else None, job['id']))
+ audit(conn, job, 'failed:' + code if failed else 'retry:' + code)
+ tele.count('job_retries', {'job_type':'image', 'outcome':'failed' if failed else 'retry'})
+ tele.log('job_failed' if failed else 'job_retry', job_id=str(job['id']), attempt=job['attempt'], error_code=code)
+
+
+def dispatch_once(client, fault=None):
+ with connect() as conn:
+ with conn.transaction():
+ expired = conn.execute("SELECT * FROM attachment_jobs WHERE 1=1 AND state = 'processing' AND lease_until < clock_timestamp() FOR UPDATE SKIP LOCKED LIMIT 20").fetchall()
+ for job in expired:
+ retry(conn, job, 'lease_expired')
+ with conn.transaction():
+ jobs = conn.execute('''
+ SELECT j.*
+ FROM attachment_jobs j
+ JOIN attachments a ON a.current_job_id = j.id
+ WHERE 1=1
+ AND j.state = 'queued'
+ AND j.next_attempt_at <= clock_timestamp()
+ AND (j.published_at IS NULL OR j.published_at < clock_timestamp() - %s * INTERVAL '1 second')
+ ORDER BY j.created_at
+ LIMIT 10
+ FOR UPDATE OF j SKIP LOCKED
+ ''', (LEASE,)).fetchall()
+ for job in jobs:
+ with tele.span('attachment publish', {'messaging.system':'redis', 'messaging.destination.name':STREAM},
+ tele.SpanKind.PRODUCER, job.get('trace_context') or {}):
+ message = event(job)
+ message.update(tele.carrier())
+ client.xadd(STREAM, {'event': json.dumps(message, separators=(',', ':'))})
+ tele.log('job_published', job_id=str(job['id']), attempt=job['attempt']+1)
+ if fault:
+ fault('after_publish', job)
+ conn.execute('UPDATE attachment_jobs SET published_at = clock_timestamp() WHERE 1=1 AND id = %s', (job['id'],))
+ audit(conn, job, 'published')
+ return len(jobs)
+
+
+def claim(message):
+ with connect() as conn, conn.transaction():
+ job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (message['job_id'],)).fetchone()
+ if job is None:
+ return None
+ attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (job['attachment_id'],)).fetchone()
+ if str(job['attachment_id']) != message['attachment_id'] or job['pipeline_version'] != message['pipeline_version']:
+ return None
+ if attachment['current_job_id'] != job['id'] or attachment['pipeline_version'] != job['pipeline_version']:
+ audit(conn, job, 'stale_noop')
+ return None
+ if job['state'] in ('ready', 'failed', 'superseded'):
+ audit(conn, job, 'terminal_noop')
+ return None
+ if job['state'] != 'queued' or job['next_attempt_at'] > dt.datetime.now(dt.timezone.utc) or message['attempt'] != job['attempt'] + 1:
+ audit(conn, job, 'duplicate_or_early_noop')
+ return None
+ token = uuid.uuid4()
+ job = conn.execute('''
+ UPDATE attachment_jobs
+ SET state = 'processing'
+ , attempt = attempt + 1
+ , lease_token = %s
+ , lease_until = clock_timestamp() + %s * INTERVAL '1 second'
+ , updated_at = clock_timestamp()
+ WHERE 1=1
+ AND id = %s
+ RETURNING *
+ ''', (token, LEASE, job['id'])).fetchone()
+ conn.execute("UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job['id'],))
+ audit(conn, job, 'claimed')
+ return job, attachment
+
+
+class Permanent(Exception):
+ pass
+
+
+def immutable(objects, key, path, mime, sha):
+ head = objects.head(key)
+ if head is not None:
+ if head['ContentLength'] != path.stat().st_size or head.get('Metadata', {}).get('sha256') != sha:
+ raise Permanent('result_conflict')
+ return
+ try:
+ with path.open('rb') as source:
+ objects.client.put_object(Bucket=objects.bucket, Key=key, Body=source, ContentLength=path.stat().st_size, ContentType=mime, Metadata={'sha256': sha}, IfNoneMatch='*')
+ except ClientError as error:
+ if error.response['ResponseMetadata']['HTTPStatusCode'] in (409, 412):
+ head = objects.head(key)
+ if head and head['ContentLength'] == path.stat().st_size and head.get('Metadata', {}).get('sha256') == sha:
+ return
+ raise
+
+
+def process_core(message, fault=None):
+ claimed = claim(message)
+ if claimed is None:
+ return 'noop'
+ job, attachment = claimed
+ objects = Objects(setting)
+ try:
+ if fault:
+ fault('before_side_effect', job)
+ with tempfile.TemporaryDirectory(prefix='job-') as temporary:
+ directory = Path(temporary)
+ source = directory / 'original'
+ head = objects.head(job['source_key'])
+ if head is None:
+ raise Permanent('source_missing')
+ response = objects.get(job['source_key'])
+ digest, size = hashlib.sha256(), 0
+ deadline = time.monotonic() + 15
+ with response['Body'] as stream, source.open('wb') as out:
+ for chunk in iter(lambda: stream.read(64 * 1024), b''):
+ size += len(chunk)
+ if size > MAX_BYTES or time.monotonic() > deadline:
+ raise Permanent('invalid_image')
+ digest.update(chunk); out.write(chunk)
+ if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']:
+ raise Permanent('checksum_mismatch')
+ result = subprocess.run([sys.executable, str(Path(__file__).with_name('render.py')), str(source), str(directory), str(job['pipeline_version'])], capture_output=True, timeout=int(os.environ.get('JOB_TIMEOUT_SECONDS', '20')))
+ if result.returncode == 2:
+ raise Permanent('invalid_image')
+ if result.returncode:
+ raise RuntimeError('render_failed')
+ variants = json.loads(result.stdout)
+ prefix = 'results/' + str(attachment['id']) + '/v' + str(job['pipeline_version']) + '/'
+ original_key = prefix + 'original'
+ immutable(objects, original_key, source, attachment['content_type'], attachment['sha256'])
+ if fault:
+ fault('after_original', job)
+ for variant in variants:
+ variant['object_key'] = prefix + variant['name']
+ immutable(objects, variant['object_key'], directory / variant['name'], variant['content_type'], variant['sha256'])
+ if fault:
+ fault('after_objects', job)
+ with connect() as conn, conn.transaction():
+ current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone()
+ changed = conn.execute('''
+ UPDATE attachments
+ SET state = 'ready'
+ , object_key = %s
+ , variants = %s
+ , error_code = NULL
+ , updated_at = clock_timestamp()
+ WHERE 1=1
+ AND current_job_id = %s
+ AND pipeline_version = %s
+ AND %s
+ RETURNING id
+ ''', (original_key, Jsonb(variants), job['id'], job['pipeline_version'], current['state'] == 'processing' and current['lease_token'] == job['lease_token'] and current['lease_until'] > dt.datetime.now(dt.timezone.utc))).fetchone()
+ if changed:
+ conn.execute("UPDATE attachment_jobs SET state = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE 1=1 AND id = %s", (job['id'],))
+ audit(conn, job, 'ready')
+ else:
+ audit(conn, job, 'fenced_noop')
+ if fault:
+ fault('after_commit', job)
+ if changed and job['source_key'].startswith('staging/'):
+ objects.delete(job['source_key'])
+ if changed:
+ tele.duration('job_ready_duration', (dt.datetime.now(dt.timezone.utc)-job['created_at']).total_seconds(), {'job_type':'image'})
+ return 'ready' if changed else 'fenced'
+ except Exception as error:
+ code = str(error) if isinstance(error, Permanent) else 'processing_timeout' if isinstance(error, subprocess.TimeoutExpired) else 'storage_unavailable' if isinstance(error, (APIError, ClientError, BotoCoreError)) else 'processing_failed'
+ with connect() as conn, conn.transaction():
+ current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone()
+ if current['state'] == 'processing' and current['lease_token'] == job['lease_token']:
+ retry(conn, current, code, isinstance(error, Permanent))
+ return 'failed_or_retry'
+ finally:
+ objects.client.close()
+
+
+def process(message, fault=None):
+ started = time.monotonic()
+ with tele.span('attachment process', {'messaging.system':'redis', 'messaging.destination.name':STREAM,
+ 'job.id':message['job_id'], 'job.attempt':message['attempt']},
+ tele.SpanKind.CONSUMER, message):
+ with tele.LOCK:
+ tele.STATE['worker_busy'] += 1
+ outcome = 'error'
+ try:
+ outcome = process_core(message, fault)
+ if outcome == 'failed_or_retry':
+ tele.trace.get_current_span().set_status(tele.Status(tele.StatusCode.ERROR))
+ return outcome
+ finally:
+ with tele.LOCK:
+ tele.STATE['worker_busy'] -= 1
+ attributes = {'job_type':'image', 'outcome':outcome}
+ tele.count('jobs_processed', attributes)
+ tele.duration('job_processing_duration', time.monotonic()-started, attributes)
+ tele.log('job_processed', job_id=message['job_id'], attempt=message['attempt'], outcome=outcome)
+
+
+def observe_queue():
+ try:
+ with raw_connect() as conn:
+ row = conn.execute("""
+ SELECT
+ count(*) AS depth
+ , coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS age
+ FROM attachment_jobs
+ WHERE 1=1
+ AND state IN ('queued', 'processing')
+ """).fetchone()
+ tele.STATE.update(queue_depth=row['depth'], queue_oldest_age=float(row['age']), queue_observation_ok=1)
+ except psycopg.Error:
+ tele.STATE['queue_observation_ok'] = 0
+
+
+def ack(client, identifier):
+ # Atomic ack+delete; PostgreSQL is the durable completion/replay registry.
+ client.eval("redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])", 1, STREAM, GROUP, identifier)
+
+
+def consume(once=False, fault=None):
+ client = broker()
+ consumer = uuid.uuid4().hex
+ while not STOP.is_set():
+ messages = []
+ try:
+ pending = client.xautoclaim(STREAM, GROUP, consumer, min_idle_time=LEASE * 1000, start_id='0-0', count=1)[1]
+ if pending:
+ messages = pending
+ else:
+ batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=500)
+ messages = batches[0][1] if batches else []
+ for identifier, fields in messages:
+ raw = fields.get('event', '')
+ try:
+ message = json.loads(raw) if len(raw) <= 4096 else None
+ except ValueError:
+ message = None
+ if validate(message):
+ process(message, fault)
+ else:
+ client.xadd('rejected-jobs', {'source_message_id': identifier, 'sha256': hashlib.sha256(raw.encode()).hexdigest(), 'reason': 'invalid_envelope'}, maxlen=1000, approximate=False)
+ ack(client, identifier)
+ if once:
+ return
+ except (redis.RedisError, psycopg.Error):
+ STOP.wait(.5)
+ if once and not messages:
+ return
+ client.close()
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('mode', choices=['dispatch', 'work'])
+ parser.add_argument('--once', action='store_true')
+ parser.add_argument('--fault-point', choices=['before_side_effect', 'after_original', 'after_objects', 'after_commit', 'after_publish'])
+ args = parser.parse_args()
+ tele.configure('messenger-dispatcher' if args.mode=='dispatch' else 'messenger-worker')
+ # Operator-only opt-in injection; no request header or public endpoint controls it.
+ def fault(point, job):
+ if point == args.fault_point:
+ os._exit(77)
+ signal.signal(signal.SIGTERM, lambda *_: STOP.set())
+ signal.signal(signal.SIGINT, lambda *_: STOP.set())
+ if args.mode == 'dispatch':
+ client = broker()
+ while not STOP.is_set():
+ observe_queue()
+ try:
+ dispatch_once(client, fault)
+ except (redis.RedisError, psycopg.Error):
+ pass
+ if args.once:
+ break
+ STOP.wait(.2)
+ else:
+ count = int(os.environ.get('WORKER_CONCURRENCY', '1'))
+ if not 1 <= count <= 2:
+ raise ValueError('WORKER_CONCURRENCY must be 1..2')
+ with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool:
+ futures = [pool.submit(consume, args.once, fault) for _ in range(count)]
+ for result in futures:
+ result.result()
+
+
+if __name__ == '__main__':
+ try:
+ main()
+ finally:
+ tele.shutdown()
diff --git a/src/postgres.py b/src/postgres.py
new file mode 100644
index 0000000..7039817
--- /dev/null
+++ b/src/postgres.py
@@ -0,0 +1,414 @@
+import base64
+import contextlib
+import datetime as dt
+import hashlib
+import hmac
+import json
+import os
+import re
+from pathlib import Path
+import secrets
+import threading
+import uuid
+import telemetry as tele
+
+import psycopg
+from psycopg.rows import dict_row
+from psycopg.types.json import Jsonb
+from psycopg_pool import ConnectionPool, PoolTimeout
+import redis
+
+from objects import Objects, unavailable
+
+from common import APIError, fields, identifier, invalid, missing, page, text, unauthorized, username
+
+KDF_SLOTS = threading.BoundedSemaphore(2)
+
+
+def setting(name):
+ path = os.environ.get(name + '_FILE')
+ return Path(path).read_text().strip() if path else os.environ[name]
+
+
+def public(row):
+ return {k: (v.decode('utf-8') if isinstance(v, bytes) else [str(x) for x in v] if isinstance(v, list) else str(v) if isinstance(v, uuid.UUID) else v.isoformat(timespec='microseconds').replace('+00:00', 'Z') if isinstance(v, dt.datetime) else v) for k, v in row.items() if k != 'password_hash'}
+
+
+def password_hash(password, salt=None):
+ salt = salt or secrets.token_bytes(16)
+ if not KDF_SLOTS.acquire(timeout=1.5):
+ raise APIError(503, 'unavailable', 'Аутентификация временно перегружена')
+ try:
+ digest = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024)
+ finally:
+ KDF_SLOTS.release()
+ return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex()
+
+
+class Store:
+ persistent = True
+
+ def __init__(self):
+ self.ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600'))
+ if not 1 <= self.ttl <= 86400:
+ raise ValueError('SESSION_TTL_SECONDS must be 1..86400')
+ self.access_mode = os.environ.get('DB_ACCESS_MODE', 'optimized')
+ if self.access_mode not in ('baseline', 'optimized'):
+ raise ValueError('DB_ACCESS_MODE must be baseline or optimized')
+ pool_size = int(os.environ.get('DB_POOL_MAX', '8'))
+ if not 1 <= pool_size <= 32:
+ raise ValueError('DB_POOL_MAX must be 1..32')
+ self.pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=pool_size, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=2000 -c lock_timeout=1500'}, open=True)
+ self.pool.wait(timeout=10)
+ tele.pool(self.pool)
+ self.cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.15, socket_timeout=0.15, decode_responses=True)
+ self.objects = Objects(setting)
+ self.dummy_hash = password_hash('dummy-password-for-unknown-users')
+ with self.connection() as conn:
+ if not conn.execute("SELECT version FROM schema_migrations WHERE 1=1 AND version = '006_observability.sql'").fetchone():
+ raise RuntimeError('Run the separate migration service first')
+
+ @contextlib.contextmanager
+ def connection(self):
+ try:
+ with tele.span('db.transaction', {'db.system':'postgresql'}, tele.SpanKind.CLIENT):
+ with self.pool.connection() as conn:
+ yield conn
+ except psycopg.errors.UniqueViolation:
+ raise APIError(409, 'conflict', 'Имя пользователя занято') from None
+ except (psycopg.Error, PoolTimeout):
+ raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None
+
+ def close(self):
+ self.pool.close()
+ self.cache.close()
+ self.objects.client.close()
+
+ def ready(self):
+ self.objects.ready()
+ with self.connection() as conn:
+ conn.execute('SELECT 1')
+
+ def register(self, body):
+ fields(body, ('username', 'display_name', 'password'))
+ name, display = username(body['username']), text(body['display_name'], 100)
+ # Passwords may contain spaces and colons; only length/UTF-8 are restricted.
+ password = body['password']
+ if not isinstance(password, str) or not 12 <= len(password) <= 128:
+ invalid()
+ try:
+ password.encode('utf-8')
+ except UnicodeError:
+ invalid()
+ hashed = password_hash(password)
+ with self.connection() as conn:
+ row = conn.execute('''
+ INSERT INTO users (id, username, display_name, password_hash)
+ VALUES (%s, %s, %s, %s)
+ RETURNING id, username, display_name, created_at
+ ''', (uuid.uuid4(), name, display.encode('utf-8'), hashed)).fetchone()
+ return public(row)
+
+ def login(self, headers):
+ authorization = headers.get('Authorization', '')
+ try:
+ scheme, encoded = authorization.split(' ', 1)
+ if scheme.lower() != 'basic':
+ unauthorized()
+ name, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
+ except (ValueError, UnicodeError):
+ unauthorized()
+ if not re_username(name) or not 12 <= len(password) <= 128:
+ unauthorized()
+ with self.connection() as conn:
+ row = conn.execute('SELECT id, username, display_name, created_at, password_hash FROM users WHERE 1=1 AND username = %s', (name,)).fetchone()
+ hashed = row['password_hash'] if row else self.dummy_hash
+ verified = hmac.compare_digest(password_hash(password, bytes.fromhex(hashed.split('$')[4])), hashed)
+ if row is None or not verified:
+ unauthorized()
+ token = secrets.token_urlsafe(32)
+ with self.connection() as conn:
+ expiry = conn.execute('''
+ INSERT INTO sessions (token_hash, user_id, expires_at)
+ VALUES (%s, %s, clock_timestamp() + %s * INTERVAL '1 second')
+ RETURNING expires_at
+ ''', (hashlib.sha256(token.encode()).hexdigest(), row['id'], self.ttl)).fetchone()['expires_at']
+ return {'token': token, 'token_type': 'Bearer', 'expires_at': public({'expires_at': expiry})['expires_at'], 'user': public(row)}
+
+ def token_hash(self, headers):
+ parts = headers.get('Authorization', '').split(' ')
+ if len(parts) != 2 or parts[0].lower() != 'bearer' or len(parts[1]) != 43 or not parts[1].isascii():
+ unauthorized()
+ return hashlib.sha256(parts[1].encode()).hexdigest()
+
+ def authenticate(self, headers):
+ token_hash = self.token_hash(headers)
+ with self.connection() as conn:
+ session = conn.execute('''
+ SELECT user_id
+ FROM sessions
+ WHERE 1=1
+ AND token_hash = %s
+ AND expires_at > clock_timestamp()
+ ''', (token_hash,)).fetchone()
+ if session is None:
+ unauthorized()
+ key = 'user:' + str(session['user_id'])
+ try:
+ cached = self.cache.get(key)
+ if cached:
+ return json.loads(cached)
+ except (redis.RedisError, ValueError):
+ pass
+ row = conn.execute('SELECT id, username, display_name, created_at FROM users WHERE 1=1 AND id = %s', (session['user_id'],)).fetchone()
+ if row is None:
+ unauthorized()
+ user = public(row)
+ try:
+ self.cache.setex(key, 60, json.dumps(user))
+ except redis.RedisError:
+ pass
+ return user
+
+ def logout(self, headers):
+ with self.connection() as conn:
+ conn.execute('DELETE FROM sessions WHERE 1=1 AND token_hash = %s', (self.token_hash(headers),))
+
+ def chat_in(self, conn, actor, chat_id):
+ row = conn.execute('''
+ SELECT c.id, c.title, c.created_at
+ FROM chats c
+ JOIN chat_members cm ON cm.chat_id = c.id
+ WHERE 1=1
+ AND c.id = %s
+ AND cm.user_id = %s
+ ''', (chat_id, actor['id'])).fetchone()
+ if row is None:
+ missing()
+ members = conn.execute('SELECT user_id FROM chat_members WHERE 1=1 AND chat_id = %s ORDER BY user_id', (chat_id,)).fetchall()
+ return dict(public(row), member_ids=[str(v['user_id']) for v in members])
+
+ def chat(self, actor, chat_id):
+ with self.connection() as conn:
+ return self.chat_in(conn, actor, chat_id)
+
+ def create_chat(self, actor, body):
+ fields(body, ('title', 'member_ids'))
+ title = text(body['title'], 100)
+ members = body['member_ids']
+ if not isinstance(members, list) or not 1 <= len(members) <= 100:
+ invalid()
+ members = [identifier(v) for v in members]
+ if len(set(members)) != len(members) or actor['id'] not in members:
+ invalid()
+ with self.connection() as conn:
+ chat_id = uuid.uuid4()
+ conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat_id, title.encode('utf-8')))
+ # Checking after INSERT exercises rollback of the whole composite operation.
+ found = conn.execute('SELECT id FROM users WHERE 1=1 AND id = ANY(%s::uuid[])', (members,)).fetchall()
+ if len(found) != len(members):
+ invalid()
+ with conn.cursor() as cur:
+ cur.executemany('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', [(chat_id, v) for v in members])
+ return self.chat_in(conn, actor, chat_id)
+
+ def list_chats(self, actor, limit, cursor, scope):
+ with self.connection() as conn:
+ where = 'AND (c.created_at, c.id) > (%s::timestamptz, %s::uuid)' if cursor else ''
+ params = [actor['id']] + list(cursor or ()) + [limit + 1]
+ rows = conn.execute('''
+ SELECT c.id
+ FROM chats c
+ JOIN chat_members cm ON cm.chat_id = c.id
+ WHERE 1=1
+ AND cm.user_id = %s
+ ''' + where + '''
+ ORDER BY c.created_at, c.id
+ LIMIT %s
+ ''', params).fetchall()
+ return page([self.chat_in(conn, actor, v['id']) for v in rows], limit, scope)
+
+ def require_member(self, conn, actor, chat_id):
+ if self.access_mode == 'baseline':
+ return self.chat_in(conn, actor, chat_id)
+ if not conn.execute('''
+ SELECT 1
+ FROM chat_members
+ WHERE 1=1
+ AND chat_id = %s
+ AND user_id = %s
+ ''', (chat_id, actor['id'])).fetchone():
+ missing()
+
+ def send(self, actor, chat_id, body, key):
+ if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
+ invalid()
+ if not isinstance(body, dict) or set(body) - {'text', 'attachment_ids'} or 'text' not in body:
+ invalid()
+ attachments = body.get('attachment_ids', [])
+ if not isinstance(attachments, list) or len(attachments) > 10:
+ invalid()
+ attachments = [identifier(v) for v in attachments]
+ if len(set(attachments)) != len(attachments):
+ invalid()
+ content = body['text']
+ if not isinstance(content, str) or len(content) > 4000 or (not attachments and not content.strip()):
+ invalid()
+ try:
+ content.encode('utf-8')
+ except UnicodeError:
+ invalid()
+ canonical = json.dumps({'text': content, 'attachment_ids': attachments}, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
+ digest = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
+ with self.connection() as conn:
+ self.require_member(conn, actor, chat_id)
+ # The unique key arbitrates concurrent processes; everything commits together.
+ claimed = conn.execute('''
+ INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, expires_at)
+ VALUES (%s, %s, %s, %s, clock_timestamp() + INTERVAL '24 hours')
+ ON CONFLICT (user_id, chat_id, key) DO UPDATE
+ SET request_hash = EXCLUDED.request_hash
+ , message_id = NULL
+ , expires_at = EXCLUDED.expires_at
+ WHERE 1=1
+ AND idempotency_keys.expires_at <= clock_timestamp()
+ RETURNING request_hash
+ ''', (actor['id'], chat_id, key, digest)).fetchone()
+ if claimed is None:
+ previous = conn.execute('''
+ SELECT ik.request_hash
+ , m.id
+ , m.chat_id
+ , m.sender_id
+ , m.text
+ , m.created_at
+ , m.attachment_ids
+ FROM idempotency_keys ik
+ JOIN messages m ON m.id = ik.message_id
+ WHERE 1=1
+ AND ik.user_id = %s
+ AND ik.chat_id = %s
+ AND ik.key = %s
+ ''', (actor['id'], chat_id, key)).fetchone()
+ if previous['request_hash'] != digest:
+ raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
+ previous.pop('request_hash')
+ return public(previous)
+ if attachments:
+ rows = conn.execute('''
+ SELECT id, state
+ FROM attachments
+ WHERE 1=1
+ AND id = ANY(%s::uuid[])
+ AND chat_id = %s
+ ORDER BY id
+ FOR SHARE
+ ''', (attachments, chat_id)).fetchall()
+ if len(rows) != len(attachments):
+ missing()
+ if any(v['state'] not in ('ready', 'queued', 'processing') for v in rows):
+ raise APIError(409, 'conflict', 'Вложение ещё не готово')
+ # Per-chat row lock preserves committed history order across replicas/clocks.
+ stamp = conn.execute('''
+ UPDATE chats
+ SET last_message_at = GREATEST(clock_timestamp(), last_message_at + INTERVAL '1 microsecond')
+ WHERE 1=1
+ AND id = %s
+ RETURNING last_message_at
+ ''', (chat_id,)).fetchone()['last_message_at']
+ row = conn.execute('''
+ INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
+ VALUES (%s, %s, %s, %s, %s, %s::uuid[])
+ RETURNING id, chat_id, sender_id, text, created_at, attachment_ids
+ ''', (uuid.uuid4(), chat_id, actor['id'], content.encode('utf-8'), stamp, attachments)).fetchone()
+ if attachments:
+ with conn.cursor() as cur:
+ cur.executemany('INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)', [(row['id'], v) for v in attachments])
+ conn.execute('''
+ UPDATE idempotency_keys
+ SET message_id = %s
+ WHERE 1=1
+ AND user_id = %s
+ AND chat_id = %s
+ AND key = %s
+ ''', (row['id'], actor['id'], chat_id, key))
+ return public(row)
+
+ def history(self, actor, chat_id, limit, cursor, scope):
+ with self.connection() as conn:
+ self.require_member(conn, actor, chat_id)
+ where = 'AND (created_at, id) > (%s::timestamptz, %s::uuid)' if cursor else ''
+ params = [chat_id] + list(cursor or ()) + [limit + 1]
+ rows = conn.execute('''
+ SELECT id, chat_id, sender_id, text, created_at, attachment_ids
+ FROM messages
+ WHERE 1=1
+ AND chat_id = %s
+ ''' + where + '''
+ ORDER BY created_at, id
+ LIMIT %s
+ ''', params).fetchall()
+ return page([public(v) for v in rows], limit, scope)
+
+ def upload(self, actor, chat_id, name, source, length, mime, digest):
+ attachment_id, job_id = uuid.uuid4(), uuid.uuid4()
+ key = 'staging/' + str(attachment_id)
+ with self.connection() as conn:
+ self.require_member(conn, actor, chat_id)
+ conn.execute('SELECT pg_advisory_xact_lock(2301005)')
+ pending = conn.execute("SELECT count(*) AS n FROM attachments WHERE 1=1 AND state IN ('uploading', 'queued', 'processing')").fetchone()['n']
+ if pending >= int(os.environ.get('MAX_PENDING_JOBS', '100')):
+ raise APIError(503, 'unavailable', 'Очередь принятых работ заполнена')
+ conn.execute('''
+ INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256, state)
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')
+ ''', (attachment_id, chat_id, actor['id'], name, 'results/' + str(attachment_id) + '/v1/original', key, mime, length, digest))
+ with self.connection() as conn:
+ if conn.execute('SELECT id FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() is None:
+ unavailable()
+ self.objects.put(key, source, length, mime, digest)
+ conn.execute('''
+ INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts, trace_context)
+ VALUES (%s, %s, 1, %s, %s, %s)
+ ''', (job_id, attachment_id, key, int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), Jsonb(tele.carrier())))
+ row = conn.execute('''
+ UPDATE attachments
+ SET state = 'queued'
+ , current_job_id = %s
+ , updated_at = clock_timestamp()
+ WHERE 1=1
+ AND id = %s
+ RETURNING *
+ ''', (job_id, attachment_id)).fetchone()
+ tele.log('attachment_accepted', attachment_id=str(attachment_id), job_id=str(job_id))
+ # Capture the queued snapshot, even if the worker finishes before HTTP reply.
+ return self.attachment_public(row)
+
+ def attachment_public(self, row):
+ keys = ('id', 'chat_id', 'owner_id', 'filename', 'content_type', 'size_bytes', 'sha256', 'created_at', 'updated_at')
+ result = public({k: row[k] for k in keys})
+ variants = [{k: v for k, v in item.items() if k != 'object_key'} for item in row['variants']]
+ return dict(result, status=row['state'], variants=variants, error_code=row['error_code'])
+
+ def attachment(self, actor, attachment_id):
+ with self.connection() as conn:
+ row = conn.execute('''
+ SELECT a.*
+ FROM attachments a
+ JOIN chat_members cm ON cm.chat_id = a.chat_id
+ WHERE 1=1
+ AND a.id = %s
+ AND cm.user_id = %s
+ ''', (attachment_id, actor['id'])).fetchone()
+ if row is None:
+ missing()
+ if row['state'] == 'storage_missing':
+ unavailable()
+ if row['state'] == 'uploading':
+ raise APIError(409, 'conflict', 'Приём не завершён')
+ return row
+
+
+def re_username(value):
+ import re
+ return re.fullmatch(r'[a-z0-9_]{3,32}', value)
diff --git a/src/render.py b/src/render.py
new file mode 100644
index 0000000..a909334
--- /dev/null
+++ b/src/render.py
@@ -0,0 +1,64 @@
+"""Isolated image decoder/resizer; no network, database or credentials."""
+import hashlib
+import json
+import math
+from pathlib import Path
+import resource
+import sys
+import warnings
+
+resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
+resource.setrlimit(resource.RLIMIT_CPU, (15, 16))
+from PIL import Image, ImageOps, PngImagePlugin
+Image.MAX_IMAGE_PIXELS = 40_000_000
+PngImagePlugin.MAX_TEXT_CHUNK = PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
+warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
+
+
+def render(source, directory, version):
+ output = []
+ with Image.open(source, formats=['PNG', 'JPEG']) as image:
+ if image.width * image.height > 40_000_000:
+ raise ValueError('pixels')
+ image.load()
+ ImageOps.exif_transpose(image, in_place=True)
+ rgba = None
+ if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info):
+ rgba = image if image.mode == 'RGBA' else image.convert('RGBA')
+ rgb = Image.new('RGB', rgba.size, 'white')
+ alpha = rgba.getchannel('A')
+ rgb.paste(rgba, mask=alpha)
+ alpha.close()
+ else:
+ rgb = image if image.mode == 'RGB' else image.convert('RGB')
+ try:
+ for name, bound, fmt, quality in [('thumbnail', 256, 'JPEG', 85), ('crop', 128, 'JPEG', 85), ('optimized', 1280, 'WEBP', 82 if version == 1 else 70)]:
+ if name == 'crop':
+ result = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(.5, .5))
+ else:
+ scale = min(1, bound / max(rgb.size))
+ size = tuple(max(1, math.floor(v * scale)) for v in rgb.size)
+ result = rgb.resize(size, Image.Resampling.LANCZOS)
+ try:
+ path = directory / name
+ options = {'quality': quality, 'exif': b'', 'icc_profile': None}
+ if fmt == 'WEBP':
+ options['method'] = 4
+ result.save(path, format=fmt, **options)
+ raw = path.read_bytes()
+ output.append({'name': name, 'width': result.width, 'height': result.height, 'content_type': 'image/jpeg' if fmt == 'JPEG' else 'image/webp', 'size_bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()})
+ finally:
+ result.close()
+ finally:
+ if rgb is not image:
+ rgb.close()
+ if rgba is not None and rgba is not image:
+ rgba.close()
+ return output
+
+
+if __name__ == '__main__':
+ try:
+ print(json.dumps(render(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3]))))
+ except (OSError, ValueError, SyntaxError, Image.DecompressionBombError, Image.DecompressionBombWarning):
+ raise SystemExit(2)
diff --git a/src/server.py b/src/server.py
new file mode 100644
index 0000000..84049a9
--- /dev/null
+++ b/src/server.py
@@ -0,0 +1,317 @@
+"""Bounded threaded API; persistent correctness state is shared across replicas."""
+import concurrent.futures
+import contextlib
+import http.server
+import json
+import logging
+import os
+import time
+import signal
+import socket
+import threading
+import uuid
+from urllib.parse import parse_qs, urlsplit
+
+from objects import bounded_upload, receive, filename, unavailable
+
+import telemetry as tele
+
+from common import APIError, identifier, invalid, missing, page_args
+
+LOG = logging.getLogger('messenger')
+
+
+def unique_object(pairs):
+ result = {}
+ for key, value in pairs:
+ if key in result:
+ invalid()
+ result[key] = value
+ return result
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+ protocol_version = 'HTTP/1.1'
+
+ def setup(self):
+ super().setup()
+ self.connection.settimeout(3)
+
+ def send_error(self, code, message=None, explain=None):
+ # BaseHTTPRequestHandler otherwise emits HTML without contract headers.
+ self.request_id = uuid.uuid4().hex
+ self.route = ''
+ self.close_connection = True
+ status = 413 if code == 413 else 400
+ self.reply(status, {'error': {'code': 'payload_too_large' if status == 413 else 'invalid_request', 'message': 'Некорректный HTTP-запрос', 'request_id': self.request_id}})
+
+ def log_message(self, *args):
+ pass
+
+ def reply(self, status, body):
+ if status == 200 and isinstance(body, tuple):
+ return self.download(*body)
+ raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8')
+ if status >= 400:
+ self.close_connection = True
+ self.send_response(status)
+ self.send_header('Content-Type', 'application/json')
+ self.send_header('Content-Length', str(len(raw)))
+ if self.close_connection:
+ self.send_header('Connection', 'close')
+ self.send_header('X-Request-Id', self.request_id)
+ self.send_header('X-Instance-Id', self.server.instance_id)
+ if status == 401 and self.server.store.persistent:
+ self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
+ self.end_headers()
+ if self.command != 'HEAD':
+ self.wfile.write(raw)
+
+ def download(self, row, response, chunk):
+ stream = response['Body']
+ try:
+ self.send_response(200)
+ self.send_header('Content-Type', row['content_type'])
+ self.send_header('Content-Length', str(row['size_bytes']))
+ self.send_header('X-Request-Id', self.request_id)
+ self.send_header('X-Instance-Id', self.server.instance_id)
+ self.send_header('X-Content-Type-Options', 'nosniff')
+ self.send_header('Cache-Control', 'private, no-store')
+ self.end_headers()
+ deadline = time.monotonic() + 15
+ while chunk:
+ self.wfile.write(chunk)
+ if time.monotonic() > deadline:
+ raise TimeoutError()
+ chunk = stream.read(64 * 1024)
+ finally:
+ stream.close()
+
+ def body(self):
+ lengths = self.headers.get_all('Content-Length', [])
+ if len(lengths) != 1 or self.headers.get('Transfer-Encoding'):
+ invalid()
+ try:
+ length = int(lengths[0])
+ except ValueError:
+ invalid()
+ if length > 32768:
+ raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт')
+ if length <= 0:
+ invalid()
+ if self.headers.get_content_type() != 'application/json':
+ raise APIError(415, 'unsupported_media_type', 'Требуется application/json')
+ try:
+ raw = self.rfile.read(length)
+ if len(raw) != length:
+ invalid()
+ return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid())
+ except (ValueError, UnicodeError, RecursionError):
+ invalid()
+
+ def dispatch(self):
+ self.request_id = uuid.uuid4().hex
+ self.route = urlsplit(self.path).path
+ template = tele.route(self.route)
+ method = self.command if self.command in ('GET','POST','DELETE','PUT','PATCH','HEAD','OPTIONS') else 'OTHER'
+ attrs = {'method':method, 'route':template}
+ started = time.monotonic()
+ context = tele.span(method+' '+template, {'http.request.method':method, 'http.route':template},
+ tele.SpanKind.SERVER, {k:self.headers[k] for k in ('traceparent','tracestate') if k in self.headers})
+ # Health probes do not enter business telemetry or create noisy dependency spans.
+ if template.startswith('/health/'):
+ context = tele.suppressed()
+ status = 500
+ with context as current:
+ try:
+ status, result = self.route_request()
+ except APIError as exc:
+ status = exc.status
+ result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}}
+ except Exception:
+ result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}}
+ try:
+ self.reply(status, result)
+ except (BrokenPipeError, ConnectionResetError, TimeoutError, OSError):
+ self.close_connection = True
+ if not template.startswith('/health/'):
+ attrs['status_class'] = str(status//100)+'xx'
+ tele.count('http_server_requests', attrs)
+ tele.duration('http_server_duration', time.monotonic()-started, attrs)
+ if current is not None:
+ current.set_attribute('http.response.status_code', status)
+ if status >= 500:
+ current.set_status(tele.Status(tele.StatusCode.ERROR))
+ tele.log('http_request', request_id=self.request_id, method=method, route=template, status=status, duration_seconds=round(time.monotonic()-started,6))
+
+ def route_request(self):
+ method, path, store = self.command, self.route, self.server.store
+ if method == 'GET' and path in ('/health/live', '/health/ready'):
+ if path.endswith('ready'):
+ store.ready()
+ return 200, {'status': 'ok'}
+ if method == 'POST' and path == '/api/v1/users':
+ return 201, store.register(self.body())
+ if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions':
+ if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
+ invalid()
+ return 201, store.login(self.headers)
+ actor = store.authenticate(self.headers)
+ if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current':
+ store.logout(self.headers)
+ return 204, None
+ if method == 'GET' and path == '/api/v1/users/me':
+ return 200, actor
+ query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
+ if path == '/api/v1/chats':
+ if method == 'POST':
+ return 201, store.create_chat(actor, self.body())
+ if method == 'GET':
+ scope = 'chats:' + actor['id']
+ limit, cursor = page_args(query, scope)
+ return 200, store.list_chats(actor, limit, cursor, scope)
+ parts = path.split('/')
+ if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'attachments'] and method == 'GET':
+ row = store.attachment(actor, identifier(parts[4]))
+ if len(parts) == 5:
+ if query:
+ invalid()
+ return 200, store.attachment_public(row)
+ if parts[5] == 'content':
+ if set(query) - {'variant'} or any(len(v) != 1 for v in query.values()):
+ invalid()
+ if row['state'] != 'ready':
+ raise APIError(409, 'conflict', 'Вложение ещё не готово')
+ variant = query.get('variant', ['original'])[0]
+ if variant != 'original':
+ item = next((v for v in row['variants'] if v['name'] == variant), None)
+ if item is None:
+ missing()
+ row = dict(row, object_key=item['object_key'], content_type=item['content_type'], size_bytes=item['size_bytes'])
+ response = store.objects.get(row['object_key'])
+ try:
+ chunk = response['Body'].read(64 * 1024)
+ if response['ContentLength'] != row['size_bytes'] or not chunk:
+ unavailable()
+ except Exception:
+ response['Body'].close()
+ unavailable()
+ return 200, (row, response, chunk)
+ if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']:
+ chat_id = identifier(parts[4])
+ if store.access_mode == 'baseline':
+ store.chat(actor, chat_id)
+ if len(parts) == 5 and method == 'GET':
+ return 200, store.chat(actor, chat_id)
+ if len(parts) == 6 and parts[5] == 'attachments' and method == 'POST':
+ store.chat(actor, chat_id)
+ if set(query) != {'filename'} or len(query['filename']) != 1:
+ invalid()
+ name = filename(query['filename'][0])
+ with bounded_upload():
+ with receive(self) as (source, length, mime, digest):
+ return 202, store.upload(actor, chat_id, name, source, length, mime, digest)
+ if len(parts) == 6 and parts[5] == 'messages':
+ if method == 'POST':
+ keys = self.headers.get_all('Idempotency-Key', [])
+ return 201, store.send(actor, chat_id, self.body(), keys[0] if len(keys) == 1 else None)
+ if method == 'GET':
+ scope = 'messages:' + chat_id
+ limit, cursor = page_args(query, scope)
+ return 200, store.history(actor, chat_id, limit, cursor, scope)
+ missing()
+
+ do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch
+
+
+class Server(http.server.HTTPServer):
+ request_queue_size = 64
+ allow_reuse_address = True
+
+ def __init__(self, address, store):
+ self.store = store
+ self.instance_id = tele.INSTANCE
+ workers = int(os.environ.get('HTTP_WORKERS', '16'))
+ if not 1 <= workers <= 128:
+ raise ValueError('HTTP_WORKERS must be 1..128')
+ self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers)
+ self.slots = threading.BoundedSemaphore(workers * 2)
+ self.connections = set()
+ self.connections_lock = threading.Lock()
+ super().__init__(address, Handler)
+
+ def process_request(self, request, address):
+ if not self.slots.acquire(blocking=False):
+ rid = uuid.uuid4().hex
+ raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode()
+ try:
+ request.settimeout(0.2)
+ request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Instance-Id: ' + self.instance_id + '\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw)
+ except OSError:
+ pass
+ self.shutdown_request(request)
+ return
+ with self.connections_lock:
+ self.connections.add(request)
+ self.executor.submit(self.worker, request, address)
+
+ def worker(self, request, address):
+ try:
+ self.finish_request(request, address)
+ except OSError:
+ pass
+ finally:
+ self.shutdown_request(request)
+ with self.connections_lock:
+ self.connections.discard(request)
+ self.slots.release()
+
+ def close_connections(self):
+ with self.connections_lock:
+ connections = list(self.connections)
+ for connection in connections:
+ try:
+ connection.shutdown(socket.SHUT_RDWR)
+ except OSError:
+ pass
+
+ def server_close(self):
+ super().server_close()
+ # Bound draining even when a slow client keeps an incomplete request alive.
+ deadline = threading.Timer(5, self.close_connections)
+ deadline.start()
+ try:
+ self.executor.shutdown(wait=True)
+ finally:
+ deadline.cancel()
+
+
+def main():
+ logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s')
+ tele.configure('messenger-api')
+ from postgres import Store
+ store = Store()
+ server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store)
+ stopping = threading.Event()
+
+ def stop(signum, frame):
+ if not stopping.is_set():
+ stopping.set()
+ LOG.info('shutdown_started signal=%s', signum)
+ threading.Thread(target=server.shutdown, daemon=True).start()
+
+ signal.signal(signal.SIGUSR1, lambda *_: LOG.warning('pool_stats instance=%s stats=%s', server.instance_id, json.dumps(store.pool.get_stats())))
+ signal.signal(signal.SIGTERM, stop)
+ signal.signal(signal.SIGINT, stop)
+ LOG.info('started port=%s', server.server_address[1])
+ try:
+ server.serve_forever(poll_interval=0.1)
+ finally:
+ server.server_close()
+ store.close()
+ tele.shutdown()
+ LOG.info('shutdown_complete')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/telemetry.py b/src/telemetry.py
new file mode 100644
index 0000000..ccc5b3a
--- /dev/null
+++ b/src/telemetry.py
@@ -0,0 +1,217 @@
+"""Allowlisted manual SDK telemetry; bounded exporters never carry business payloads."""
+import contextlib
+import contextvars
+import datetime as dt
+import json
+import logging
+import os
+from pathlib import Path
+import re
+import resource
+import threading
+import time
+import uuid
+
+from opentelemetry import trace, metrics
+from opentelemetry.trace import SpanKind, Status, StatusCode
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider, SpanProcessor
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult
+from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+
+INSTANCE = uuid.uuid4().hex
+ACTIVE = False
+SUPPRESSED = contextvars.ContextVar('telemetry_suppressed', default=False)
+PROVIDERS = []
+COUNTERS = {}
+HISTOGRAMS = {}
+STATE = {'worker_busy': 0, 'queue_depth': 0, 'queue_oldest_age': 0, 'queue_observation_ok': 1,
+ 'spans_finished': 0, 'spans_exported': 0, 'spans_export_failed': 0}
+LOCK = threading.Lock()
+PROPAGATOR = TraceContextTextMapPropagator()
+LOGGER = logging.getLogger('messenger.telemetry')
+TRACER = trace.get_tracer('messenger')
+POOL = None
+SERVICE = ''
+BUCKETS = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 60]
+
+class FinishedSpans(SpanProcessor):
+ def on_end(self, span):
+ with LOCK:
+ STATE['spans_finished'] += 1
+
+class CountedExporter(SpanExporter):
+ def __init__(self, delegate):
+ self.delegate = delegate
+
+ def export(self, spans):
+ result = self.delegate.export(spans)
+ with LOCK:
+ STATE['spans_exported' if result == SpanExportResult.SUCCESS else 'spans_export_failed'] += len(spans)
+ return result
+
+ def shutdown(self):
+ self.delegate.shutdown()
+
+ def force_flush(self, timeout_millis=30000):
+ return self.delegate.force_flush(timeout_millis)
+
+def configure(service):
+ global ACTIVE, TRACER, SERVICE
+ if PROVIDERS or os.environ.get('TELEMETRY_ENABLED', '1') == '0':
+ return
+ SERVICE = service
+ ratio = float(os.environ.get('TRACE_SAMPLE_RATIO', '0.25'))
+ if not 0 <= ratio <= 1:
+ raise ValueError('TRACE_SAMPLE_RATIO must be 0..1')
+ resource_attributes = Resource.create({
+ 'service.name': service, 'service.version': 'lab6.1',
+ 'service.instance.id': INSTANCE, 'deployment.environment.name': 'lab',
+ })
+ endpoint = os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
+ provider = TracerProvider(resource=resource_attributes, sampler=ParentBased(TraceIdRatioBased(ratio)))
+ provider.add_span_processor(FinishedSpans())
+ provider.add_span_processor(BatchSpanProcessor(
+ CountedExporter(OTLPSpanExporter(endpoint=endpoint+'/v1/traces', timeout=1)),
+ max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=500,
+ export_timeout_millis=1500))
+ trace.set_tracer_provider(provider)
+ TRACER = trace.get_tracer('messenger')
+ reader = PeriodicExportingMetricReader(
+ OTLPMetricExporter(endpoint=endpoint+'/v1/metrics', timeout=1),
+ export_interval_millis=2000, export_timeout_millis=1500)
+ meter_provider = MeterProvider(resource=resource_attributes, metric_readers=[reader],
+ views=[View(instrument_name='*duration', aggregation=ExplicitBucketHistogramAggregation(BUCKETS))])
+ metrics.set_meter_provider(meter_provider)
+ meter = metrics.get_meter('messenger')
+ for name in ['http_server_requests', 'jobs_processed', 'job_retries']:
+ COUNTERS[name] = meter.create_counter(name)
+ for name in ['http_server_duration', 'job_processing_duration', 'job_ready_duration']:
+ HISTOGRAMS[name] = meter.create_histogram(name, unit='s')
+ def gauge(name, getter, unit=''):
+ def observe(options):
+ try:
+ yield metrics.Observation(getter(), {})
+ except (OSError, ValueError):
+ return
+ meter.create_observable_gauge(name, callbacks=[observe], unit=unit)
+ for name in ['queue_depth', 'queue_oldest_age', 'queue_observation_ok', 'worker_busy',
+ 'spans_finished', 'spans_exported', 'spans_export_failed']:
+ gauge(name, lambda name=name: STATE[name], 's' if name=='queue_oldest_age' else '')
+ gauge('worker_capacity', lambda: int(os.environ.get('WORKER_CONCURRENCY', '1')) if service=='messenger-worker' else 0)
+ gauge('db_pool_used', lambda: max(0, POOL.get_stats().get('pool_size',0)-POOL.get_stats().get('pool_available',0)) if POOL else 0)
+ gauge('db_pool_waiting', lambda: POOL.get_stats().get('requests_waiting',0) if POOL else 0)
+ gauge('db_pool_capacity', lambda: int(os.environ.get('DB_POOL_MAX','8')) if POOL else 0)
+ gauge('process_resident_memory', lambda: int(Path('/proc/self/statm').read_text().split()[1])*os.sysconf('SC_PAGE_SIZE'), 'By')
+ gauge('container_memory', lambda: int(Path('/sys/fs/cgroup/memory.current').read_text()), 'By')
+ def cpu(options):
+ use = resource.getrusage(resource.RUSAGE_SELF)
+ yield metrics.Observation(use.ru_utime+use.ru_stime, {})
+ meter.create_observable_counter('process_cpu', callbacks=[cpu], unit='s')
+ log_provider = LoggerProvider(resource=resource_attributes)
+ log_provider.add_log_record_processor(BatchLogRecordProcessor(
+ OTLPLogExporter(endpoint=endpoint+'/v1/logs', timeout=1),
+ max_queue_size=256, max_export_batch_size=64, schedule_delay_millis=500,
+ export_timeout_millis=1500))
+ LOGGER.setLevel(logging.INFO)
+ LOGGER.propagate = False
+ LOGGER.addHandler(LoggingHandler(level=logging.INFO, logger_provider=log_provider))
+ PROVIDERS.extend([provider, meter_provider, log_provider])
+ ACTIVE = True
+
+def pool(value):
+ global POOL
+ POOL = value
+
+def route(path):
+ if path in ['/health/live', '/health/ready', '/api/v1/users', '/api/v1/users/me',
+ '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats']:
+ return path
+ patterns = [
+ (r'/api/v1/chats/[^/]+/messages', '/api/v1/chats/{chat_id}/messages'),
+ (r'/api/v1/chats/[^/]+/attachments', '/api/v1/chats/{chat_id}/attachments'),
+ (r'/api/v1/chats/[^/]+', '/api/v1/chats/{chat_id}'),
+ (r'/api/v1/attachments/[^/]+/content', '/api/v1/attachments/{attachment_id}/content'),
+ (r'/api/v1/attachments/[^/]+', '/api/v1/attachments/{attachment_id}'),
+ ]
+ return next((template for pattern,template in patterns if re.fullmatch(pattern,path)), 'unmatched')
+
+@contextlib.contextmanager
+def span(name, attributes=None, kind=SpanKind.INTERNAL, carrier=None):
+ if not ACTIVE or SUPPRESSED.get() or (name == 'db.transaction' and not trace.get_current_span().get_span_context().is_valid):
+ yield trace.get_current_span()
+ return
+ context = PROPAGATOR.extract(carrier) if carrier is not None else None
+ with TRACER.start_as_current_span(name, attributes=attributes or {}, kind=kind,
+ context=context, record_exception=False, set_status_on_exception=False) as current:
+ try:
+ yield current
+ except Exception as error:
+ current.set_attribute('error.type', type(error).__name__)
+ current.set_status(Status(StatusCode.ERROR))
+ log('operation_failed', error_type=type(error).__name__)
+ raise
+
+@contextlib.contextmanager
+def suppressed():
+ token = SUPPRESSED.set(True)
+ try:
+ yield None
+ finally:
+ SUPPRESSED.reset(token)
+
+def carrier():
+ result = {}
+ if ACTIVE:
+ PROPAGATOR.inject(result)
+ return result
+
+def log(event, **attributes):
+ if not ACTIVE:
+ return
+ # Callers pass only fixed event names, bounded outcomes and technical correlation IDs.
+ context = trace.get_current_span().get_span_context()
+ body = {'timestamp': dt.datetime.now(dt.timezone.utc).isoformat(),
+ 'severity': 'INFO', 'service.name': SERVICE, 'event': event,
+ 'sampled': bool(context.trace_flags.sampled), 'trace_id': format(context.trace_id,'032x'), 'span_id': format(context.span_id,'016x'),
+ **attributes}
+ LOGGER.info(json.dumps(body, ensure_ascii=True, separators=(',',':')))
+
+def count(name, attributes=None):
+ if ACTIVE:
+ COUNTERS[name].add(1, attributes or {})
+
+def duration(name, seconds, attributes=None):
+ if ACTIVE:
+ HISTOGRAMS[name].record(max(0, seconds), attributes or {})
+
+def flush():
+ for provider in PROVIDERS:
+ provider.force_flush(timeout_millis=5000)
+
+def shutdown():
+ for provider in PROVIDERS:
+ provider.shutdown()
+
+class S3Client:
+ """Span each SDK request without serializing args, keys, URLs or SDK exceptions."""
+ def __init__(self, client):
+ self.client = client
+
+ def __getattr__(self, name):
+ method = getattr(self.client,name)
+ if name not in {'head_object','get_object','put_object','delete_object','head_bucket','create_bucket','list_objects_v2'}:
+ return method
+ def call(*args, **kwargs):
+ with span('s3.'+name, {'rpc.system':'aws-api','rpc.service':'S3','rpc.method':name}, SpanKind.CLIENT):
+ return method(*args, **kwargs)
+ return call
diff --git a/student-tests/durability.py b/student-tests/durability.py
new file mode 100644
index 0000000..3470de0
--- /dev/null
+++ b/student-tests/durability.py
@@ -0,0 +1,52 @@
+"""Stop worker, accept queued, restart API/broker, then complete the same attachment."""
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
+from client import Client,Failure
+
+
+def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip()
+
+
+def main():
+ c=Client();alice,bob,eve,chat=c.bootstrap();record={}
+ before=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True))
+ try:
+ docker('compose','stop','worker','dispatcher')
+ queued,raw=c.attachment(alice,chat)
+ assert queued['status']=='queued'
+ message=c.message(alice,chat,'accepted with worker stopped',attachments=[queued['id']])['json']
+ c.call('GET','/api/v1/attachments/'+queued['id']+'/content',409,actor=bob)
+ killed=docker('compose','ps','-q','api').splitlines()
+ for container in killed:docker('kill','--signal=KILL',container)
+ record['killed_api_containers']=killed
+ docker('compose','up','-d','--wait','--no-deps','api')
+ time.sleep(2)
+ assert c.call('GET','/api/v1/attachments/'+queued['id'],actor=bob,schema='Attachment')['json']['status']=='queued'
+ docker('compose','up','-d','--no-deps','dispatcher')
+ deadline=time.monotonic()+10
+ while int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))<1:
+ assert time.monotonic()=length_before>=1
+ record['stream_entries_before_after_broker_restart']=[length_before,length_after]
+ docker('compose','up','-d','--no-deps','worker','dispatcher')
+ ready=c.wait_ready(bob,queued)
+ assert c.call('GET','/api/v1/attachments/'+queued['id']+'/content',actor=bob)['bytes']==raw
+ record.update(attachment_id=queued['id'],original_sha256=hashlib.sha256(raw).hexdigest(),status=ready['status'],variants=[v['name'] for v in ready['variants']],old_bearer_survived=True,message_attachment_retained=message['attachment_ids']==[queued['id']])
+ finally:docker('compose','up','-d','--no-deps','worker','dispatcher')
+ after=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True))
+ record['before']=before;record['after']=after
+ print(json.dumps(record,indent=2))
+
+
+if __name__=='__main__':main()
diff --git a/student-tests/images.py b/student-tests/images.py
new file mode 100644
index 0000000..906356f
--- /dev/null
+++ b/student-tests/images.py
@@ -0,0 +1,24 @@
+"""Small deterministic PNG helpers; generation happens outside measurement."""
+import random
+import struct
+import zlib
+
+
+def chunk(name, data):
+ return struct.pack('!I', len(data)) + name + data + struct.pack('!I', zlib.crc32(name + data) & 0xffffffff)
+
+
+def png(width, height, noise=False, rgb=True):
+ channels = 3 if rgb else 1
+ rand = random.Random(2301)
+ encoded = zlib.compressobj(1)
+ parts = []
+ for _ in range(height):
+ line = bytes(rand.getrandbits(8) for _ in range(width * channels)) if noise else b'\x7f' * (width * channels)
+ parts.append(encoded.compress(b'\0' + line))
+ parts.append(encoded.flush())
+ return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2 if rgb else 0, 0, 0, 0)) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'')
+
+
+def bomb():
+ return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', 10000, 4001, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'x')) + chunk(b'IEND', b'')
diff --git a/student-tests/observability.py b/student-tests/observability.py
new file mode 100644
index 0000000..5b8a624
--- /dev/null
+++ b/student-tests/observability.py
@@ -0,0 +1,257 @@
+"""Real OTLP evidence, controlled incidents and bounded telemetry-backend outage."""
+import argparse
+import base64
+import concurrent.futures
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+from urllib.parse import urlencode
+import urllib.request
+import uuid
+
+ROOT=Path(__file__).resolve().parents[1]
+sys.path.insert(0,str(ROOT/'tests'))
+from client import Client, Failure
+
+BASE=os.environ.get('BASE_URL','http://127.0.0.1:18093')
+PROM=os.environ.get('PROMETHEUS_URL','http://127.0.0.1:19096')
+LOKI=os.environ.get('LOKI_URL','http://127.0.0.1:13106')
+TEMPO=os.environ.get('TEMPO_URL','http://127.0.0.1:13206')
+COLLECTOR=os.environ.get('COLLECTOR_URL','http://127.0.0.1:18896')
+OPENER=urllib.request.build_opener(urllib.request.ProxyHandler({}))
+
+def get(url):
+ req=urllib.request.Request(url,headers={'Accept':'application/json'})
+ with OPENER.open(req,timeout=10) as response:
+ return json.load(response)
+
+def prom(query):
+ return get(PROM+'/api/v1/query?'+urlencode({'query':query}))['data']['result']
+
+def scalar(query):
+ rows=prom(query)
+ return float(rows[0]['value'][1]) if rows else 0
+
+def logs(query, start=None, limit=2000):
+ params={'query':query,'start':str(int((start or time.time()-600)*1e9)),
+ 'end':str(time.time_ns()),'limit':str(limit),'direction':'forward'}
+ result=get(LOKI+'/loki/api/v1/query_range?'+urlencode(params))['data']['result']
+ return [json.loads(value) for stream in result for stamp,value in stream['values']]
+
+def trace(identifier):
+ return get(TEMPO+'/api/traces/'+identifier)
+
+def spans(data):
+ result=[]
+ for batch in data.get('batches',data.get('resourceSpans',[])):
+ service=next(a['value']['stringValue'] for a in batch['resource']['attributes'] if a['key']=='service.name')
+ for scope in batch.get('scopeSpans',[]):
+ for span in scope['spans']:
+ result.append(dict(span,service=service))
+ return result
+
+def wait(function,timeout=60):
+ end=time.monotonic()+timeout
+ while time.monotonic()=4 else None)
+ assert {r['event'] for r in records}>={'http_request','attachment_accepted','job_published','job_processed'}
+ assert any(r.get('request_id')==response['headers'].get('X-Request-Id') for r in records)
+ assert all(len(r['trace_id'])==32 and len(r['span_id'])==16 for r in records)
+ all_logs=logs('{service_name=~"messenger-.*"}',start=proof_start,limit=5000)
+ payload=json.dumps(all_logs)+json.dumps(data)
+ for value in [marker,a['token'],a['registration']['password'],a['registration']['username'],'private-canary.png']:
+ assert value not in payload, 'business/credential canary leaked'
+ metric_rows=prom('{service_name=~"messenger-.*"}')
+ forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','job_id','trace_id','span_id'}
+ assert all(not forbidden.intersection(row['metric']) for row in metric_rows)
+ names={r['metric']['__name__'] for r in metric_rows}
+ assert {'http_server_duration_seconds_bucket','jobs_processed_total','queue_depth','queue_oldest_age_seconds','db_pool_used','worker_capacity'}<=names
+ save('trace-proof.json',{'trace_id':tid,'request_id':response['headers'].get('X-Request-Id'),
+ 'status':ready['status'],'consumer_parent_matches_producer':True,'producer_descends_from_upload':True,
+ 'api_worker_logs_correlated':True,'secret_canaries_absent':True,'metric_labels_allowlisted':True,
+ 'trace':data,'logs':records})
+ # Retry is a real S3 outage after durable acceptance, not fabricated spans.
+ compose('stop','worker')
+ retry_tid=uuid.uuid4().hex
+ queued=upload(c,a,chat,retry_tid)['json']
+ paused=False
+ try:
+ compose('pause','s3');paused=True
+ compose('up','-d','--no-deps','worker')
+ def attempted():
+ raw=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',
+ "SELECT attempt FROM attachment_jobs WHERE 1=1 AND attachment_id = '"+queued['id']+"';")
+ return int(raw)>=1
+ wait(attempted)
+ time.sleep(5)
+ finally:
+ if paused:compose('unpause','s3')
+ compose('up','-d','--no-deps','worker')
+ wait(lambda:c.call('GET','/health/ready',schema='Health'))
+ c.wait_ready(a,queued)
+ def retried():
+ data=trace(retry_tid)
+ return data if len([s for s in spans(data) if s['kind']=='SPAN_KIND_CONSUMER'])>=2 else None
+ retry_data=wait(retried)
+ retry_records=wait(lambda:logs('{service_name="messenger-worker"} | json | trace_id="'+retry_tid+'" | event="job_retry"'))
+ save('retry-trace.json',{'trace_id':retry_tid,'retry_observed':True,'trace':retry_data,'logs':retry_records})
+ print('PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.')
+
+def alerts():
+ return get(PROM+'/api/v1/alerts')['data']['alerts']
+
+def firing(name):
+ return [a for a in alerts() if a['labels']['alertname']==name and a['state']=='firing']
+
+def incidents():
+ c=Client(BASE);a,b,e,chat=c.bootstrap();result={}
+ start=time.time()
+ compose('stop','worker')
+ try:
+ queued=[c.attachment(a,chat)[0] for _ in range(3)]
+ observed=wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')>=3)
+ fired=wait(lambda:firing('QueueLag'))
+ detected=time.time()
+ metrics={q:prom(q) for q in ['queue_depth','queue_oldest_age_seconds','worker_busy','worker_capacity']}
+ result['worker_stop']={'injected_at':start,'detected_at':detected,
+ 'detection_seconds':detected-start,'firing':fired,'metrics':metrics,
+ 'diagnosis':'Backlog and oldest age grow; worker completion stops while API acceptance still succeeds. Gauges expire after 30s.'}
+ finally:
+ compose('up','-d','--no-deps','worker')
+ for att in queued:c.wait_ready(a,att)
+ wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')==0)
+ wait(lambda:not firing('QueueLag'))
+ result['worker_stop']['resolved_at']=time.time()
+ # Keep readiness SELECT 1 available while real session SQL is blocked.
+ lock=subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1',
+ '-c',"BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(35); COMMIT;"],
+ stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+ time.sleep(1)
+ db_start=time.time();stop=False;errors=[];error_trace=uuid.uuid4().hex
+ def traffic():
+ while not stop:
+ r=c.call('GET','/api/v1/users/me',[200,503],actor=a,
+ headers={'traceparent':'00-'+error_trace+'-0123456789abcdef-01'})
+ if r['status']==503:errors.append(r['elapsed'])
+ try:
+ with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
+ futures=[pool.submit(traffic) for _ in range(4)]
+ try:
+ fired=wait(lambda:firing('APIHighErrorRate'),timeout=50)
+ detected=time.time()
+ result['db_lock']={'injected_at':db_start,'detected_at':detected,'detection_seconds':detected-db_start,
+ 'firing':fired,'error_latencies_seconds':errors.copy(),'trace_id':error_trace}
+ lock.wait(timeout=45)
+ finally:
+ stop=True
+ for future in futures:future.result()
+ finally:
+ stop=True
+ if lock.poll() is None:
+ lock.terminate();lock.wait(timeout=5)
+ wait(lambda:c.call('GET','/api/v1/users/me',actor=a,schema='User'))
+ wait(lambda:not firing('APIHighErrorRate'),timeout=70)
+ data=wait(lambda:trace(error_trace))
+ assert any(s['name']=='db.transaction' and s.get('status',{}).get('code')=='STATUS_CODE_ERROR' for s in spans(data))
+ result['db_lock'].update(resolved_at=time.time(),trace=data,
+ diagnosis='Server 5xx spans contain slow/error PostgreSQL transaction; health remains available; sessions lock is the bottleneck.')
+ save('incidents.json',result)
+ print('PASS: worker-stop and real DB-lock incidents, local firing/resolved and causal DB traces.')
+
+def collector_metrics():
+ with OPENER.open(COLLECTOR+'/metrics',timeout=5) as r:raw=r.read().decode()
+ result={}
+ for line in raw.splitlines():
+ if line.startswith('otelcol_') and not line.startswith('#'):
+ name=line.split('{')[0].split(' ')[0]
+ try:result[name]=result.get(name,0)+float(line.rsplit(' ',1)[-1])
+ except ValueError:pass
+ return result
+
+def backend_outage():
+ c=Client(BASE);a,b,e,chat=c.bootstrap();before=collector_metrics()
+ cid=compose('ps','-q','otel-collector')
+ samples=[];messages=[];attachments=[];start=time.time()
+ compose('stop','tempo')
+ try:
+ for i in range(45):
+ for _ in range(3):
+ messages.append(c.message(a,chat,'synthetic-backend-outage')['json']['id'])
+ if i%10==0:
+ att,_=c.attachment(a,chat);attachments.append(c.wait_ready(a,att)['id'])
+ stat=json.loads(subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',cid],text=True))
+ samples.append({'elapsed':time.time()-start,'memory':stat['MemUsage'],'cpu':stat['CPUPerc']})
+ time.sleep(1)
+ during=collector_metrics()
+ assert during.get('otelcol_exporter_send_failed_spans_total',0)>before.get('otelcol_exporter_send_failed_spans_total',0)
+ assert during.get('otelcol_exporter_queue_size',0)<=128*2
+ history=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100',actor=b,schema='MessagePage')['json']
+ got={m['id'] for m in history['items']}
+ if history['next_cursor']:
+ history2=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+history['next_cursor'],actor=b,schema='MessagePage')['json']
+ got.update(m['id'] for m in history2['items'])
+ assert set(messages)<=got
+ finally:compose('up','-d','--no-deps','tempo')
+ wait(lambda:get(TEMPO+'/api/search'))
+ tid=uuid.uuid4().hex
+ upload(c,a,chat,tid)
+ wait(lambda:trace(tid))
+ save('backend-outage.json',{'duration_seconds':time.time()-start,'confirmed_messages':len(messages),
+ 'confirmed_attachments_ready':len(attachments),'all_confirmed_messages_retained':True,
+ 'collector_samples':samples,'before':before,'during':during,'recovered_trace_id':tid})
+ print('PASS: Tempo unavailable, finite Collector buffers/drops, API writes/jobs retained, fresh trace after recovery.')
+
+if __name__=='__main__':
+ parser=argparse.ArgumentParser()
+ parser.add_argument('phase',choices=['proof','incidents','backend'])
+ args=parser.parse_args()
+ {'proof':proof,'incidents':incidents,'backend':backend_outage}[args.phase]()
diff --git a/student-tests/observability_cost.py b/student-tests/observability_cost.py
new file mode 100644
index 0000000..830402e
--- /dev/null
+++ b/student-tests/observability_cost.py
@@ -0,0 +1,193 @@
+"""Fixed-arrival mixed workload: SDK off, full traces, 10% and 25% head sampling."""
+import concurrent.futures
+import http.client
+import json
+import os
+from pathlib import Path
+import socket
+import statistics
+import subprocess
+import sys
+import threading
+import time
+from urllib.parse import urlsplit
+import uuid
+
+sys.path.insert(0,str(Path(__file__).resolve().parent))
+from observability import Client, BASE, compose, collector_metrics, scalar, prom, logs, wait, save
+
+class DockerHTTP(http.client.HTTPConnection):
+ def connect(self):
+ self.sock=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM)
+ self.sock.settimeout(8)
+ self.sock.connect(DOCKER_SOCKET)
+
+context=json.loads(subprocess.check_output(['docker','context','inspect'],text=True))[0]
+DOCKER_SOCKET=urlsplit(context['Endpoints']['docker']['Host']).path
+
+def stats(container):
+ connection=DockerHTTP('localhost')
+ try:
+ connection.request('GET','/containers/'+container+'/stats?stream=false&one-shot=true')
+ response=connection.getresponse()
+ assert response.status==200
+ data=json.load(response)
+ finally:connection.close()
+ memory=data.get('memory_stats',{})
+ details=memory.get('stats',{})
+ usage=memory.get('usage',0)
+ return {'cpu_seconds':data['cpu_stats']['cpu_usage']['total_usage']/1e9,
+ 'anonymous_bytes':details.get('anon',details.get('rss',0)),
+ 'working_set_bytes':max(0,usage-details.get('inactive_file',details.get('total_inactive_file',0))),
+ 'rx_bytes':sum(n.get('rx_bytes',0) for n in data.get('networks',{}).values())}
+
+def span_counts():
+ return {r['metric']['service_instance_id']:float(r['value'][1])
+ for r in prom('spans_finished{service_name=~"messenger-.*"}')}
+
+def delta(previous,current):
+ return sum(max(0,value-previous.get(key,0)) for key,value in current.items())
+
+def quantile(values,q):
+ return sorted(values)[min(len(values)-1,int((len(values)-1)*q))]
+
+def set_mode(enabled,ratio):
+ env=os.environ.copy()
+ env.update(TELEMETRY_ENABLED=str(enabled),TRACE_SAMPLE_RATIO=str(ratio))
+ subprocess.run(['docker','compose','up','-d','--wait','--no-deps','api','worker','dispatcher'],env=env,
+ check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+ c=Client(BASE)
+ wait(lambda:c.call('GET','/health/ready',schema='Health'))
+ return c
+
+def probes(c,actor,mode):
+ results=[]
+ for duration in [3.5,3.5,3.5,1.2]:
+ query="BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep("+str(duration)+"); COMMIT;"
+ lock=subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',query],
+ stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+ try:
+ def locked():
+ value=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',
+ "SELECT count(*) FROM pg_locks WHERE 1=1 AND relation = 'sessions'::regclass AND mode = 'AccessExclusiveLock' AND granted;")
+ return int(value)>0
+ wait(locked,timeout=5)
+ result=c.call('GET','/api/v1/users/me',[200,503],actor=actor)
+ assert result['status']==(503 if duration>3 else 200)
+ assert result['elapsed']>.1
+ results.append({'request_id':result['headers'].get('X-Request-Id'),
+ 'status':result['status'],'latency_seconds':result['elapsed']})
+ finally:lock.wait(timeout=5)
+ if mode!='off':
+ time.sleep(3)
+ for row in results:
+ records=logs('{service_name="messenger-api"} | json | request_id="'+row['request_id']+'"',limit=10)
+ assert records
+ row['trace_sampled']=records[0]['sampled']
+ row['trace_id']=records[0]['trace_id']
+ else:
+ for row in results:row['trace_sampled']=False
+ return results
+
+def run(mode,enabled,ratio,repeat,seconds,rps):
+ c=set_mode(enabled,ratio);a,b,e,chat=c.bootstrap()
+ for _ in range(10):c.call('GET','/api/v1/users/me',actor=a)
+ time.sleep(5)
+ names=compose('ps','-q','api','worker','dispatcher','otel-collector','postgres','loki','tempo','prometheus','grafana','s3','s3-store','broker','lb','redis').splitlines()
+ labels={}
+ for cid in names:
+ label=json.loads(subprocess.check_output(['docker','inspect',cid,'--format','{{json .Config.Labels}}'],text=True))
+ labels[cid]=label['com.docker.compose.service']
+ before={cid:stats(cid) for cid in names}
+ ingest_before=collector_metrics();spans_before=span_counts()
+ memory={cid:[] for cid in names};sampling_stop=threading.Event()
+ def sample():
+ while not sampling_stop.is_set():
+ for cid in names:
+ try:memory[cid].append(stats(cid))
+ except OSError:pass
+ sampling_stop.wait(1)
+ sampler=threading.Thread(target=sample,daemon=True);sampler.start()
+ latencies=[];failures=[];attachments=[];start=time.monotonic()
+ def request(i):
+ try:
+ if i%20==0:
+ result=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=cost.png',202,
+ actor=a,raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes(),
+ headers={'Content-Type':'image/png'},schema='Attachment')
+ return result['elapsed'],result['json']
+ if i%5==0:
+ result=c.message(a,chat,'fixed-synthetic-cost-payload')
+ else:result=c.call('GET','/api/v1/users/me',actor=a,schema='User')
+ return result['elapsed'],None
+ except Exception as error:return None,type(error).__name__
+ with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool:
+ futures=[]
+ for i in range(seconds*rps):
+ delay=start+i/rps-time.monotonic()
+ if delay>0:time.sleep(delay)
+ futures.append(pool.submit(request,i))
+ for future in futures:
+ elapsed,att=future.result()
+ if elapsed is None:failures.append(att)
+ else:
+ latencies.append(elapsed)
+ if att:attachments.append(att)
+ load_wall=time.monotonic()-start
+ for att in attachments:c.wait_ready(a,att)
+ # Drain exporter batches and metric-reader intervals, then compare counters by instance.
+ time.sleep(5)
+ sampling_stop.set();sampler.join(timeout=15)
+ after={cid:stats(cid) for cid in names}
+ ingest_after=collector_metrics();spans_after=span_counts()
+ assert not failures, failures
+ resource={}
+ for cid in names:
+ service=labels[cid]
+ target=resource.setdefault(service,{'cpu_seconds':0,'peak_sampled_anonymous_bytes':0,'peak_sampled_working_set_bytes':0,'rx_bytes':0})
+ target['cpu_seconds']+=max(0,after[cid]['cpu_seconds']-before[cid]['cpu_seconds'])
+ target['rx_bytes']+=max(0,after[cid]['rx_bytes']-before[cid]['rx_bytes'])
+ target['peak_sampled_anonymous_bytes']+=max((s['anonymous_bytes'] for s in memory[cid]),default=0)
+ target['peak_sampled_working_set_bytes']+=max((s['working_set_bytes'] for s in memory[cid]),default=0)
+ counts={k:max(0,ingest_after.get(k,0)-ingest_before.get(k,0)) for k in [
+ 'otelcol_receiver_accepted_spans_total','otelcol_receiver_accepted_log_records_total',
+ 'otelcol_receiver_accepted_metric_points_total','otelcol_exporter_sent_spans_total',
+ 'otelcol_exporter_send_failed_spans_total']}
+ finished=delta(spans_before,spans_after)
+ accepted=counts['otelcol_receiver_accepted_spans_total']
+ result={'mode':mode,'enabled':enabled,'sample_ratio':ratio,'repeat':repeat,
+ 'arrival_rps':rps,'scheduled_requests':seconds*rps,'confirmed_requests':len(latencies),
+ 'load_wall_seconds':load_wall,'successful_rps':len(latencies)/load_wall,
+ 'p50_seconds':quantile(latencies,.5),'p95_seconds':quantile(latencies,.95),'p99_seconds':quantile(latencies,.99),
+ 'errors':failures,'image_jobs_ready':len(attachments),'resource':resource,'collector_ingest':counts,
+ 'sampled_spans_finished':finished,'sampled_spans_not_accepted_after_drain':max(0,finished-accepted)}
+ result['rare_probes']=probes(c,a,mode)
+ print(mode,repeat,'p95',round(result['p95_seconds'],5),'ingested spans',accepted,flush=True)
+ return result
+
+def main():
+ import argparse
+ parser=argparse.ArgumentParser()
+ parser.add_argument('--seconds',type=int,default=20)
+ parser.add_argument('--rps',type=int,default=30)
+ parser.add_argument('--repeats',type=int,default=3)
+ args=parser.parse_args()
+ result={'method':'Fixed arrival mixed requests, 5% image uploads, 15% messages, 80% profile reads; same fixture/resources. No transparent retries.',
+ 'notes':['CPU cgroup total across load plus exporter drain; includes renderer.',
+ 'Anonymous/working-set memory sampled at 1Hz through read-only Docker stats API, not exact per-PID peak RSS.',
+ 'Collector rx bytes include OTLP HTTP, Prometheus scrapes and backend replies; accepted signal counts are exact Collector counter deltas.',
+ 'Fresh API/worker/dispatcher containers per setting; data/backends retained. Per-instance counter deltas ignore expired old series.',
+ 'Rare SQL-lock probes run after resource/ingest measurement; three SQL-lock probes and one slow success per run; future runs synchronize on pg_locks.'],
+ 'runs':[]}
+ modes=[('off',0,0),('full',1,1),('sample10',1,.1),('sample25',1,.25)]
+ try:
+ for repeat in range(args.repeats):
+ # Rotate order to reduce a fixed warm-cache/time-order advantage.
+ order=modes[repeat%len(modes):]+modes[:repeat%len(modes)]
+ for mode,enabled,ratio in order:
+ result['runs'].append(run(mode,enabled,ratio,repeat,args.seconds,args.rps))
+ save('observability-cost.json',result)
+ finally:set_mode(1,.25)
+ print('PASS: telemetry cost experiment completed; balanced 25% tracing restored.')
+
+if __name__=='__main__':main()
diff --git a/student-tests/persistence_telemetry.py b/student-tests/persistence_telemetry.py
new file mode 100644
index 0000000..ffb48a1
--- /dev/null
+++ b/student-tests/persistence_telemetry.py
@@ -0,0 +1,37 @@
+"""Restart all stores and verify historical telemetry alongside public business persistence."""
+import json
+import os
+from pathlib import Path
+import subprocess
+import time
+from urllib.parse import urlencode
+from observability import BASE,PROM,trace,spans,logs,get,wait,save
+
+root=Path(__file__).resolve().parents[1]
+identifier=json.loads((root/'evidence/trace-proof.json').read_text())['trace_id']
+before_trace=trace(identifier)
+before_ids={s['spanId'] for s in spans(before_trace)}
+query='{service_name=~"messenger-.*"} | json | trace_id="'+identifier+'"'
+before_logs=logs(query,start=time.time()-3600)
+assert before_logs
+at=time.time()-5
+url=PROM+'/api/v1/query?'+urlencode({'query':'sum(http_server_requests_total)','time':str(at)})
+before_metric=get(url)['data']['result']
+assert before_metric
+command=['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh','BASE_URL='+BASE]
+result=subprocess.run(command,capture_output=True,text=True)
+(root/'evidence/persistence.txt').write_text(result.stdout+result.stderr)
+assert result.returncode==0,result.stdout[-300:]
+after_trace=wait(lambda:trace(identifier),timeout=90)
+after_ids={s['spanId'] for s in spans(after_trace)}
+assert before_ids<=after_ids
+after_logs=wait(lambda:logs(query,start=time.time()-3600),timeout=90)
+assert {json.dumps(r,sort_keys=True) for r in before_logs}<={json.dumps(r,sort_keys=True) for r in after_logs}
+after_metric=wait(lambda:get(url)['data']['result'])
+assert before_metric==after_metric
+save('telemetry-persistence.json',{'trace_id':identifier,'historical_span_count':len(before_ids),
+ 'historical_logs':len(before_logs),'fixed_metric_query_time':at,
+ 'metric_before':before_metric,'metric_after':after_metric,
+ 'business_data_and_existing_session_survived':True,'all_telemetry_volumes_restarted':True,
+ 'trace_log_metric_history_survived':True})
+print('PASS: full business restart, original Bearer, historical Tempo trace, Loki logs and fixed-time Prometheus sample.')
diff --git a/student-tests/pipeline_checks.py b/student-tests/pipeline_checks.py
new file mode 100644
index 0000000..c4d36ba
--- /dev/null
+++ b/student-tests/pipeline_checks.py
@@ -0,0 +1,188 @@
+"""Real DB/S3/Redis fault injection. Stop production worker/dispatcher first."""
+import concurrent.futures
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+import uuid
+
+sys.path[:0]=['/app/src',str(Path.cwd()/'tests')]
+from PIL import Image
+from client import Client
+from pipeline import STREAM,GROUP,broker,connect,event,dispatch_once,process,ack
+from objects import Objects
+from postgres import setting
+from admin import replay,version,cleanup
+
+
+def main():
+ c=Client();alice,bob,eve,chat=c.bootstrap()
+ bus=broker(); objects=Objects(setting)
+ report={'invariants':{},'faults':[]}
+ def fetch(aid):
+ with connect() as conn:
+ a=conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s',(aid,)).fetchone()
+ j=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(a['current_job_id'],)).fetchone()
+ return a,j
+ def upload(raw=None,mime='image/png'):
+ if raw is None: a,_=c.attachment(alice,chat)
+ else:a=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=research.png',202,actor=alice,raw=raw,headers={'Content-Type':mime},schema='Attachment')['json']
+ return a,fetch(a['id'])[1]
+ def run_once(fault=None):
+ args=[sys.executable,'/app/src/pipeline.py','work','--once']
+ if fault:args+=['--fault-point',fault]
+ return subprocess.run(args,capture_output=True,timeout=30).returncode
+ def drain():
+ for _ in range(200):
+ batch=bus.xreadgroup(GROUP,'research-drain',{STREAM:'>'},count=1)
+ if not batch:break
+ for mid,data in batch[0][1]:process(json.loads(data['event']));ack(bus,mid)
+ def expire(job):
+ with connect() as conn:
+ conn.execute("UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE 1=1 AND id = %s",(job['id'],))
+ def recover(aid):
+ until=time.monotonic()+20
+ while time.monotonic() no publisher yet: one queued row, no Redis message, then automatic outbox pass.
+ a,j=upload()
+ assert fetch(a['id'])[0]['state']=='queued'
+ assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',409,actor=bob)
+ c.message(bob,chat,'queued is linkable',attachments=[a['id']])
+ dispatch_once(bus);drain();assert fetch(a['id'])[0]['state']=='ready'
+ report['invariants']['commit_before_publish_recovered']=True
+ # Broker accepted XADD but publisher died before marking outbox: true exit(77).
+ a,j=upload()
+ code=subprocess.run([sys.executable,'/app/src/pipeline.py','dispatch','--once','--fault-point','after_publish'],capture_output=True,timeout=10).returncode
+ assert code==77
+ dispatch_once(bus);drain()
+ assert fetch(a['id'])[0]['state']=='ready'
+ report['invariants']['publish_before_marker_duplicate_safe']=True
+ for point in ['before_side_effect','after_original','after_objects','after_commit']:
+ a,j=upload();dispatch_once(bus)
+ code=run_once(point);assert code==77,(point,code)
+ before,job=fetch(a['id'])
+ if point not in ('after_commit','before_side_effect'):expire(job)
+ result,job=recover(a['id'])
+ assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',actor=bob)['bytes']==(Path.cwd()/'tests/fixtures/sample.png').read_bytes()
+ reclaimed=bus.xautoclaim(STREAM,GROUP,'post-crash-redelivery',min_idle_time=0,start_id='0-0',count=100)[1]
+ for mid,data in reclaimed:
+ process(json.loads(data['event']));ack(bus,mid)
+ assert bus.xpending(STREAM,GROUP)['pending']==0
+ keys=objects.client.list_objects_v2(Bucket=objects.bucket,Prefix='results/'+a['id']+'/v1/').get('Contents',[])
+ assert len(keys)==4,(point,len(keys))
+ report['faults'].append({'point':point,'exit_code':code,'state_at_crash':before['state'],'recovered':result['state'],'attempts':job['attempt'],'result_objects':len(keys)})
+ report['invariants']['crashes_before_after_side_effect_and_commit']=True
+ bus.xadd(STREAM,{'event':json.dumps({'not':'a job'})})
+ assert run_once()==0
+ assert bus.xlen('rejected-jobs')>=1
+ report['invariants']['invalid_envelope_observable_dlq']=True
+ # Duplicate/reorder through broker on a completed job: no extra attempt/effect.
+ a,j=upload();dispatch_once(bus);drain();before,done=fetch(a['id'])
+ old=event(j)
+ for attempt in [2,1,3,1,1]:bus.xadd(STREAM,{'event':json.dumps(dict(old,attempt=attempt))})
+ drain();after,done2=fetch(a['id'])
+ assert before['variants']==after['variants'] and done2['attempt']==done['attempt']
+ report['invariants']['duplicate_and_reorder_no_second_effect']=True
+ # Race two consumers with the SAME logical job, not just one serial duplicate.
+ a,j=upload();e=event(j)
+ with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: results=list(pool.map(lambda _:process(e),range(2)))
+ assert fetch(a['id'])[0]['state']=='ready' and sorted(results)==['noop','ready']
+ report['invariants']['concurrent_duplicate_single_claim']=True
+ # Pause actual old process after all immutable side effects, advance version and finish v2.
+ a,j=upload();e=event(j)
+ directory=Path('/tmp')/('fence-'+uuid.uuid4().hex);directory.mkdir();marker=directory/'paused';release=directory/'release'
+ code="import sys,json,time;from pathlib import Path;sys.path.insert(0,'/app/src');from pipeline import process\ndef hook(point,job):\n if point=='after_objects':\n Path(sys.argv[2]).write_text('paused')\n while not Path(sys.argv[3]).exists():time.sleep(.05)\nprint(process(json.loads(sys.argv[1]),hook))"
+ child=subprocess.Popen([sys.executable,'-c',code,json.dumps(e),str(marker),str(release)],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
+ try:
+ deadline=time.monotonic()+20
+ while not marker.exists():
+ assert child.poll() is None
+ assert time.monotonic()=250
+ transforms.append({'kind':kind,'size':thumb.size,'exif_removed':not bool(thumb.getexif())})
+ report['transforms']=transforms
+ pending,pending_job=upload()
+ dry=cleanup(False);fixed=cleanup(True);repeat=cleanup(True)
+ assert fetch(pending['id'])[0]['state']=='queued' and objects.head(pending_job['source_key']) is not None
+ report['invariants']['cleanup_preserves_current_and_fresh_failed']=True
+ assert repeat['counts']['ready_staging']==0
+ report['invariants']['ready_staging_cleanup_idempotent']=True
+ report['cleanup']=[dry,fixed,repeat]
+ print(json.dumps(report,indent=2))
+
+
+if __name__=='__main__': main()
diff --git a/student-tests/plot.py b/student-tests/plot.py
new file mode 100644
index 0000000..157828d
--- /dev/null
+++ b/student-tests/plot.py
@@ -0,0 +1,28 @@
+"""Figures from real accepted/ready and cgroup samples, no synthetic results."""
+import json
+from pathlib import Path
+import statistics
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+r=Path(__file__).resolve().parents[1]/'evidence'
+x=json.loads((r/'resources.json').read_text())['experiments']
+fig,axes=plt.subplots(1,2,figsize=(11,4.3),layout='constrained')
+labels=['320×200\n1,3 KiB','1800×1400\n7,21 MiB','10000×4000\n0,51 MiB']
+a=[max(v['accepted_seconds'] for v in e['replies']) for e in x]
+b=[max(v['ready_seconds'] for v in e['replies']) for e in x]
+axes[0].bar([i-.18 for i in range(3)],a,.36,label='p95 до 202')
+axes[0].bar([i+.18 for i in range(3)],b,.36,label='p95 до ready')
+axes[0].set(xticks=range(3),xticklabels=labels,ylabel='Секунды',title='10 одновременных загрузок · worker=1')
+axes[0].legend()
+peak=[int(e['after']['memory.peak'])/1048576 for e in x]
+anon=[max(int(dict(line.split() for line in s['worker']['memory.stat'].splitlines())['anon']) for s in e['samples'])/1048576 for e in x]
+axes[1].bar([i-.18 for i in range(3)],peak,.36,label='cgroup memory.peak')
+axes[1].bar([i+.18 for i in range(3)],anon,.36,label='Пик resident anon')
+axes[1].axhline(768,color='#d62728',linestyle='--',label='Лимит worker')
+axes[1].set(xticks=range(3),xticklabels=labels,ylabel='MiB',title='Worker + дочерний декодер')
+axes[1].legend(fontsize=8)
+for ax in axes:ax.grid(axis='y',alpha=.2);ax.set_axisbelow(True)
+fig.suptitle('ЛР 5 · реальные измерения · concurrency=1, prefetch=1')
+fig.savefig(r/'pipeline.png',dpi=150)
+fig.savefig(r/'pipeline.svg')
diff --git a/student-tests/plot_observability.py b/student-tests/plot_observability.py
new file mode 100644
index 0000000..9c7e956
--- /dev/null
+++ b/student-tests/plot_observability.py
@@ -0,0 +1,36 @@
+"""Publication plot from measured cost/retention JSON; matplotlib is research-only."""
+import json
+from pathlib import Path
+import statistics
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+
+ROOT=Path(__file__).resolve().parents[1]
+data=json.loads((ROOT/'evidence/observability-cost.json').read_text())
+modes=['off','full','sample10','sample25']
+labels=['Off','Full 100%','Sample 10%','Sample 25%']
+colors=['#64748b','#2563eb','#0d9488','#d97706']
+fig,axs=plt.subplots(2,3,figsize=(16,9),layout='constrained')
+def draw(ax,values,title,ylabel):
+ med=[statistics.median(v) for v in values]
+ low=[m-min(v) for m,v in zip(med,values)]
+ high=[max(v)-m for m,v in zip(med,values)]
+ ax.bar(range(4),med,color=colors,yerr=[low,high],capsize=4,alpha=.85)
+ for i,v in enumerate(values):ax.scatter([i]*len(v),v,color='#0f172a',s=14,zorder=3)
+ ax.set_xticks(range(4),labels,rotation=12)
+ ax.set_title(title,loc='left',fontweight='bold')
+ ax.set_ylabel(ylabel)
+ ax.grid(axis='y',alpha=.2)
+ ax.set_axisbelow(True)
+rows=[[r for r in data['runs'] if r['mode']==mode] for mode in modes]
+draw(axs[0,0],[[r['p95_seconds']*1000 for r in group] for group in rows],'HTTP p95','milliseconds')
+draw(axs[0,1],[[r['resource']['api']['cpu_seconds'] for r in group] for group in rows],'API CPU (both replicas)','CPU seconds / run + drain')
+draw(axs[0,2],[[r['collector_ingest']['otelcol_receiver_accepted_spans_total'] for r in group] for group in rows],'Accepted traces','spans / run + drain')
+draw(axs[1,0],[[r['resource']['otel-collector']['rx_bytes']/1024 for r in group] for group in rows],'Collector network received','KiB (includes scrape / protocol traffic)')
+draw(axs[1,1],[[r['resource']['api']['peak_sampled_anonymous_bytes']/1048576 for r in group] for group in rows],'API anonymous memory','MiB (sum of 1 Hz sampled replica peaks)')
+draw(axs[1,2],[[r['resource']['worker']['cpu_seconds'] for r in group] for group in rows],'Worker CPU incl renderer','CPU seconds / run + drain')
+fig.suptitle('Lab 6: cost of observability | 600 mixed requests per run (30 uploads) | 30 req/s | 3 repeats',fontsize=15,fontweight='bold')
+fig.savefig(ROOT/'evidence/observability-cost.png',dpi=150)
+fig.savefig(ROOT/'evidence/observability-cost.svg')
+print('Plot written from actual per-run JSON; dots = repeats, bars = median, whiskers = min/max.')
diff --git a/student-tests/resources.py b/student-tests/resources.py
new file mode 100644
index 0000000..a9c409d
--- /dev/null
+++ b/student-tests/resources.py
@@ -0,0 +1,52 @@
+"""Finite simultaneous jobs including 40 MP; record actual worker cgroup peak/RSS."""
+import concurrent.futures
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+import sys
+import threading
+import time
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
+from client import Client
+from images import png
+
+
+def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip()
+
+
+def memory(container):
+ code="import json;from pathlib import Path;p=Path('/sys/fs/cgroup');print(json.dumps({k:(p/k).read_text().strip() for k in ['memory.current','memory.peak','memory.events','memory.stat','cpu.stat']}))"
+ return json.loads(docker('exec',container,'python','-c',code))
+
+
+def main():
+ c=Client(timeout=30);a,b,e,chat=c.bootstrap()
+ profiles=[(320,200,False),(1800,1400,True),(10000,4000,False)]
+ result=[]
+ for w,h,noise in profiles:
+ content=png(w,h,noise=noise)
+ docker('compose','restart','worker');container=docker('compose','ps','-q','worker')
+ before=memory(container);samples=[];done=threading.Event();start=time.monotonic()
+ def sample():
+ while not done.is_set():
+ samples.append({'seconds':round(time.monotonic()-start,4),'worker':memory(container)})
+ done.wait(.05)
+ sampler=threading.Thread(target=sample);sampler.start()
+ barrier=threading.Barrier(10)
+ def upload(i):
+ barrier.wait();t=time.monotonic()
+ r=Client(timeout=30).call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=load.png',202,actor=a,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
+ accepted=time.monotonic()-t
+ ready=Client(timeout=30).wait_ready(b,r['json'])
+ assert ready['sha256']==hashlib.sha256(content).hexdigest()
+ return {'index':i,'accepted_seconds':accepted,'ready_seconds':time.monotonic()-t,'status':ready['status']}
+ try:
+ with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: replies=list(pool.map(upload,range(10)))
+ finally:done.set();sampler.join()
+ after=memory(container);assert 'oom_kill 0' in after['memory.events']
+ result.append({'width':w,'height':h,'bytes':len(content),'clients':10,'before':before,'after':after,'samples':samples,'replies':replies,'seconds':time.monotonic()-start})
+ print(json.dumps({'method':'10 simultaneous uploads, worker concurrency=1; cgroup v2 samples include child renderer; anon is resident anonymous memory, memory.peak includes tmpfs/kernel/cache','experiments':result},indent=2))
+
+
+if __name__=='__main__':main()
diff --git a/student-tests/sampling_retention.py b/student-tests/sampling_retention.py
new file mode 100644
index 0000000..a8e7dda
--- /dev/null
+++ b/student-tests/sampling_retention.py
@@ -0,0 +1,38 @@
+"""Separate synchronized rare-error/slow-success retention experiment."""
+import json
+from pathlib import Path
+from observability_cost import set_mode, probes
+from observability import save, trace, wait
+import urllib.error
+
+
+def verify_presence(rows):
+ for row in rows:
+ if row["trace_sampled"]:
+ data=wait(lambda:trace(row["trace_id"]),timeout=90)
+ assert data.get("batches",data.get("resourceSpans"))
+ row["tempo_trace_found"]=True
+ else:
+ try:
+ trace(row["trace_id"])
+ except urllib.error.HTTPError as error:
+ assert error.code==404
+ row["tempo_trace_found"]=False
+ else:
+ raise AssertionError("Unsampled trace unexpectedly stored")
+ return rows
+
+def main():
+ result={'method':'Three repeats per full/10%/25% setting; each holds sessions lock until pg_locks confirms it. 3x503 plus one slow200 per repeat. Sampling is ordinary SDK head sampling without forced traceparent.','runs':[]}
+ try:
+ for mode,ratio in [('full',1),('sample10',.1),('sample25',.25)]:
+ c=set_mode(1,ratio)
+ actor=c.user('retention')
+ for repeat in range(3):
+ result['runs'].append({'mode':mode,'repeat':repeat,'probes':verify_presence(probes(c,actor,mode))})
+ save('sampling-retention.json',result)
+ print(mode,repeat,'retention recorded',flush=True)
+ finally:set_mode(1,.25)
+ print('PASS: synchronized rare 503/slow200 retention study; balanced sampling restored.')
+
+if __name__=='__main__':main()
diff --git a/student-tests/security.py b/student-tests/security.py
new file mode 100644
index 0000000..7fb5321
--- /dev/null
+++ b/student-tests/security.py
@@ -0,0 +1,29 @@
+"""Private S3 and job envelopes do not expose source bytes or credentials."""
+import json
+from pathlib import Path
+import sys
+import urllib.error
+import urllib.request
+sys.path[:0]=['/app/src',str(Path.cwd()/'tests')]
+from client import Client
+from pipeline import connect,validate,event
+from objects import Objects
+from postgres import setting
+
+c=Client();a,b,e,chat=c.bootstrap();att,raw=c.attachment(a,chat);ready=c.wait_ready(b,att)
+with connect() as conn:
+ row=conn.execute('SELECT object_key, current_job_id FROM attachments WHERE 1=1 AND id = %s',(att['id'],)).fetchone()
+ job=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(row['current_job_id'],)).fetchone()
+message=event(job);message['attempt']=max(1,job['attempt'])
+assert validate(message)
+required={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'}
+assert required <= set(message) <= required | {'traceparent','tracestate'}
+assert validate(message)
+assert len(json.dumps(message))<1024
+for path in ('/'+Objects(setting).bucket+'/'+row['object_key'],'/'+Objects(setting).bucket+'?list-type=2'):
+ try:urllib.request.urlopen('http://s3:8333'+path,timeout=2);raise AssertionError('anonymous access')
+ except urllib.error.HTTPError as error:assert error.code==403
+for key in ['schema_version','pipeline_version','attempt']:
+ bad=dict(message);bad[key]=True;assert not validate(bad)
+for body in [dict(message,extra='x'),dict(message,created_at='bad'),dict(message,traceparent='bad')]:assert not validate(body)
+print(json.dumps({'anonymous_object_and_list_403':True,'job_fields':sorted(message),'job_bytes':len(json.dumps(message)),'no_source_bytes_or_credentials_in_envelope':True,'strict_invalid_envelope_checks':True},indent=2))
diff --git a/student-tests/slow_upload.py b/student-tests/slow_upload.py
new file mode 100644
index 0000000..c17d2b0
--- /dev/null
+++ b/student-tests/slow_upload.py
@@ -0,0 +1,32 @@
+"""A continuously trickling client still reaches the absolute upload deadline."""
+import json
+from pathlib import Path
+import select
+import socket
+import sys
+import time
+import uuid
+from urllib.parse import urlsplit
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
+from client import Client
+
+c = Client()
+alice, bob, eve, chat = c.bootstrap()
+target = urlsplit(c.base)
+connection = socket.create_connection((target.hostname, target.port), timeout=3)
+header = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=slow_' + uuid.uuid4().hex + '.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n'
+started = time.monotonic()
+connection.sendall(header.encode())
+response = b''
+while time.monotonic() - started < 20:
+ if select.select([connection], [], [], .15)[0]:
+ response += connection.recv(4096)
+ if b'\r\n\r\n' in response:
+ break
+ connection.sendall(b'x')
+connection.close()
+elapsed = time.monotonic() - started
+assert response.startswith(b'HTTP/1.1 400') or response.startswith(b'HTTP/1.0 400'), response[:100]
+assert 14 <= elapsed < 19
+assert b'x-instance-id:' in response.lower() and b'x-request-id:' in response.lower()
+print(json.dumps({'trickle_seconds': round(elapsed, 3), 'http_status': 400, 'absolute_deadline_enforced': True}))
diff --git a/student-tests/test_api.py b/student-tests/test_api.py
new file mode 100644
index 0000000..45f0d80
--- /dev/null
+++ b/student-tests/test_api.py
@@ -0,0 +1,177 @@
+"""Extra black-box tests, without changing the course's public tests."""
+import concurrent.futures
+from pathlib import Path
+import sys
+import unittest
+from urllib.parse import quote
+import uuid
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
+from client import Client as PublicClient, LAB
+
+
+class Client(PublicClient):
+ def call(self, method, path, *args, headers=None, **kwargs):
+ headers = dict(headers or {})
+ if method == 'POST' and path.endswith('/messages'):
+ headers.setdefault('Idempotency-Key', uuid.uuid4().hex)
+ return super().call(method, path, *args, headers=headers, **kwargs)
+
+
+class APIContract(unittest.TestCase):
+ def setUp(self):
+ self.client = Client()
+ self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
+ self.path = '/api/v1/chats/' + self.chat['id']
+
+ def test_malformed_json_and_extra_fields(self):
+ for raw in (b'{', b'[]', b'{"text":"x","text":"y"}', b'{"text":NaN}', b'{"text":"\\ud800"}'):
+ self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'})
+ for body in ({'text': 'ok', 'sender_id': self.eve['id']}, {'text': True}, {'text': ' '}, {'text': ''}, {'text': None}):
+ self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body=body)
+ deep = b'{"text":' + b'[' * 1200 + b'0' + b']' * 1200 + b'}'
+ self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=deep, headers={'Content-Type': 'application/json'})
+
+ def test_unicode_limits(self):
+ result = self.client.message(self.alice, self.chat, '👩' * 4000)['json']
+ self.assertEqual(result['text'], '👩' * 4000)
+ self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': '👩' * 4001})
+
+ def test_unicode_nul_round_trip(self):
+ import base64
+ value = 'A\x00B\\u0000👩'
+ registration = dict(self.alice['registration'], username='nul_' + uuid.uuid4().hex[:20], display_name=value)
+ user = self.client.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json']
+ self.assertEqual(user['display_name'], value)
+ credential = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
+ session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json']
+ self.assertEqual(session['user'], user)
+ actor = {'id': user['id'], 'token': session['token']}
+ # Exercise the database path and then the Redis profile cache.
+ for _ in range(2):
+ self.assertEqual(self.client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'], user)
+ chat = self.client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': value, 'member_ids': [actor['id']]}, schema='Chat')['json']
+ self.assertEqual(chat['title'], value)
+ self.assertEqual(self.client.call('GET', '/api/v1/chats/' + chat['id'], actor=actor, schema='Chat')['json'], chat)
+ contents = [value, '\x00' * 4000, '👩' * 3999 + '\x00']
+ for content in contents:
+ self.assertEqual(self.client.message(actor, chat, content)['json']['text'], content)
+ history = self.client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor, schema='MessagePage')['json']
+ self.assertEqual([message['text'] for message in history['items']], contents)
+
+ def test_foreign_chat(self):
+ for suffix in ('', '/messages'):
+ self.client.call('GET', self.path + suffix, 404, actor=self.eve)
+ self.client.call('POST', self.path + '/messages', 404, actor=self.eve, body={'text': 'hidden'})
+ self.client.call('GET', '/api/v1/chats/' + str(uuid.uuid4()), 404, actor=self.alice)
+
+ def test_registration_race(self):
+ body = dict(self.alice['registration'], username='race_' + uuid.uuid4().hex[:20])
+ def register(_):
+ return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status']
+ with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+ statuses = list(executor.map(register, range(10)))
+ self.assertEqual(statuses.count(201), 1)
+ self.assertEqual(statuses.count(409), 9)
+
+ def test_invalid_chat_creation(self):
+ members = ([self.alice['id'], self.alice['id']], [self.bob['id']], [self.alice['id'], str(uuid.uuid4())], [], [True])
+ for member_ids in members:
+ self.client.call('POST', '/api/v1/chats', 400, actor=self.alice, body={'title': 'rollback', 'member_ids': member_ids})
+
+ def test_cursor_scope_and_validation(self):
+ for i in range(3):
+ self.client.message(self.alice, self.chat, str(i))
+ cursor = self.client.call('GET', self.path + '/messages?limit=1', actor=self.alice, schema='MessagePage')['json']['next_cursor']
+ other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'other', 'member_ids': [self.alice['id']]}, schema='Chat')['json']
+ self.client.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(cursor), 400, actor=self.alice)
+ for query in ('limit=0', 'limit=101', 'limit=' + '9' * 5000, 'limit=true', 'limit=1&limit=2', 'cursor=', 'cursor=bad', 'cursor=e30=', 'offset=1'):
+ self.client.call('GET', self.path + '/messages?' + query, 400, actor=self.alice)
+
+ def test_concurrent_messages_and_pagination(self):
+ def send(i):
+ return Client().message(self.alice if i % 2 else self.bob, self.chat, 'parallel-' + str(i))['json']
+ with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+ sent = list(executor.map(send, range(100)))
+ found = []
+ cursor = None
+ while True:
+ url = self.path + '/messages?limit=7' + ('&cursor=' + quote(cursor) if cursor else '')
+ page = self.client.call('GET', url, actor=self.alice, schema='MessagePage')['json']
+ found.extend(page['items'])
+ cursor = page['next_cursor']
+ if cursor is None:
+ break
+ self.assertEqual(len(found), 100)
+ self.assertEqual(len({v['id'] for v in found}), 100)
+ self.assertEqual({v['id'] for v in found}, {v['id'] for v in sent})
+ self.assertEqual(found, sorted(found, key=lambda v: (v['created_at'], v['id'])))
+
+ def test_idempotency_concurrent_replicas(self):
+ key = uuid.uuid4().hex
+ def send(_):
+ return Client().message(self.alice, self.chat, 'Один раз\x00👩', key=key)
+ with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
+ replies = list(executor.map(send, range(32)))
+ self.assertGreaterEqual(len({v['headers']['X-Instance-Id'] for v in replies}), 2)
+ self.assertTrue(all(v['json'] == replies[0]['json'] for v in replies))
+ page = self.client.call('GET', self.path + '/messages', actor=self.bob, schema='MessagePage')['json']
+ self.assertEqual(page['items'], [replies[0]['json']])
+ self.client.call('POST', self.path + '/messages', 409, actor=self.alice, body={'text': 'different'}, headers={'Idempotency-Key': key})
+ self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=self.alice)
+ self.client.call('POST', self.path + '/messages', 401, actor=self.alice, body={'text': 'Один раз\x00👩'}, headers={'Idempotency-Key': key})
+
+ def test_protocol_error_headers(self):
+ self.client.call('TRACE', '/api/v1/users/me', 400, actor=self.alice)
+
+ def test_idempotency_scope(self):
+ key = uuid.uuid4().hex
+ one = self.client.message(self.alice, self.chat, 'same', key=key)['json']
+ two = self.client.message(self.bob, self.chat, 'same', key=key)['json']
+ other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'scope', 'member_ids': [self.alice['id']]}, schema='Chat')['json']
+ three = self.client.message(self.alice, other, 'same', key=key)['json']
+ self.assertEqual(len({one['id'], two['id'], three['id']}), 3)
+
+ def test_idempotency_semantic_body(self):
+ key = uuid.uuid4().hex
+ one = self.client.message(self.alice, self.chat, 'semantic', key=key)['json']
+ two = self.client.call('POST', self.path + '/messages', 201, actor=self.alice, body={'attachment_ids': [], 'text': 'semantic'}, headers={'Idempotency-Key': key}, schema='Message')['json']
+ self.assertEqual(one, two)
+
+ def test_idempotency_invalid_keys(self):
+ for key in (None, '', 'x' * 129, 'space key', 'bad,key', 'slash/key'):
+ headers = {} if key is None else {'Idempotency-Key': key}
+ PublicClient().call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': 'valid'}, headers=headers)
+ for raw in (b'{"text":true}', b'{"text":"ok","extra":1}', b'{"text":"ok","attachment_ids":false}'):
+ self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'})
+
+ def test_idempotency_conflicting_race(self):
+ key = uuid.uuid4().hex
+ def send(i):
+ return Client().call('POST', self.path + '/messages', [201, 409], actor=self.alice, body={'text': str(i % 2)}, headers={'Idempotency-Key': key}, schema=None)
+ with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
+ replies = list(executor.map(send, range(32)))
+ self.assertEqual(sum(v['status'] == 201 for v in replies), 16)
+ self.assertEqual(sum(v['status'] == 409 for v in replies), 16)
+ page = self.client.call('GET', self.path + '/messages', actor=self.alice, schema='MessagePage')['json']
+ self.assertEqual(len(page['items']), 1)
+
+ @unittest.skipUnless(LAB >= 2, 'sessions start in lab2')
+ def test_unicode_password_basic_and_logout(self):
+ import base64
+ registration = {'username': 'unicode_' + uuid.uuid4().hex[:16], 'display_name': 'Софья', 'password': '🔐' * 127 + ':'}
+ self.client.call('POST', '/api/v1/users', 201, body=registration)
+ encoded = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
+ headers = {'Authorization': 'Basic ' + encoded}
+ session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers=headers, schema='Session')['json']
+ actor = {'id': session['user']['id'], 'token': session['token']}
+ self.client.call('GET', '/api/v1/users/me', actor=actor)
+ self.client.call('GET', '/api/v1/users/me', 401, headers=headers)
+ for authorization in ('Basic !!!', 'Bearer wrong', '', 'Basic Zm9v'):
+ self.client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': authorization})
+ self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
+ self.client.call('GET', '/api/v1/users/me', 401, actor=actor)
+
+
+if __name__ == '__main__':
+ unittest.main(verbosity=2)
diff --git a/student-tests/test_images.py b/student-tests/test_images.py
new file mode 100644
index 0000000..f27c039
--- /dev/null
+++ b/student-tests/test_images.py
@@ -0,0 +1,52 @@
+"""Independent tests for async image metadata, transformations and access."""
+import io
+from pathlib import Path
+import sys
+import unittest
+import uuid
+from urllib.parse import quote
+sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'tests')]
+from client import Client
+from images import bomb
+
+
+class Images(unittest.TestCase):
+ def setUp(self):
+ self.c=Client(); self.alice,self.bob,self.eve,self.chat=self.c.bootstrap()
+ self.path='/api/v1/chats/'+self.chat['id']
+ self.raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes()
+
+ def test_async_variants_and_replay(self):
+ queued,_=self.c.attachment(self.alice,self.chat)
+ self.assertEqual(queued['status'],'queued')
+ key=uuid.uuid4().hex
+ one=self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json']
+ self.assertEqual(self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json'],one)
+ ready=self.c.wait_ready(self.alice,queued)
+ self.assertEqual({v['name'] for v in ready['variants']},{'thumbnail','crop','optimized'})
+ dimensions={v['name']:(v['width'],v['height']) for v in ready['variants']}
+ self.assertEqual(dimensions,{'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)})
+ for v in ready['variants']:
+ import hashlib
+ r=self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant='+v['name'],actor=self.bob)
+ self.assertEqual(hashlib.sha256(r['bytes']).hexdigest(),v['sha256'])
+ self.assertEqual(len(r['bytes']),v['size_bytes'])
+ self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant=absent',404,actor=self.bob)
+
+ def test_mime_bytes_pixels_and_filename(self):
+ path=self.path+'/attachments?filename=test.png'
+ for raw,mime,status in [(b'not image','image/png',415),(self.raw[:20],'image/png',415),(self.raw,'image/jpeg',415),(bomb(),'image/png',413),(b'x'*(10485760+1),'image/png',413)]:
+ self.c.call('POST',path,status,actor=self.alice,raw=raw,headers={'Content-Type':mime})
+ for name in ('../x.png','a\\x.png','a\0.png','a\n.png','a'*256,''):
+ self.c.call('POST',self.path+'/attachments?filename='+quote(name,safe=''),400,actor=self.alice,raw=self.raw,headers={'Content-Type':'image/png'})
+
+ def test_private_and_cross_chat(self):
+ att,_=self.c.attachment(self.alice,self.chat)
+ for suffix in ('','/content'):
+ self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=self.eve)
+ self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,401)
+ other=self.c.call('POST','/api/v1/chats',201,actor=self.alice,body={'title':'other','member_ids':[self.alice['id']]},schema='Chat')['json']
+ self.c.call('POST','/api/v1/chats/'+other['id']+'/messages',404,actor=self.alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
+
+
+if __name__=='__main__': unittest.main(verbosity=2)