diff --git a/.dockerignore b/.dockerignore index bc16e55..8eb0c08 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,5 @@ __pycache__ node_modules evidence .DS_Store + +build diff --git a/.env.example b/.env.example index 2b32655..b3abae7 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,19 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +APP_PORT=18093 +GRAFANA_PORT=13006 +PROMETHEUS_PORT=19096 +LOKI_PORT=13106 +TEMPO_PORT=13206 +COLLECTOR_PORT=18896 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +API_MEMORY=256m +HTTP_WORKERS=16 +DB_POOL_MAX=8 +DB_ACCESS_MODE=optimized +WORKER_CONCURRENCY=1 +MAX_PENDING_JOBS=100 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +JOB_TIMEOUT_SECONDS=20 +JOB_LEASE_SECONDS=60 +TELEMETRY_ENABLED=1 +TRACE_SAMPLE_RATIO=0.25 diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 0000000..11cfb04 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,24 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + BASE_URL: http://127.0.0.1:18093 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python3 scripts/bootstrap.py + - run: docker compose build api + - run: docker compose up -d --wait + - run: python3 scripts/wait_ready.py + - run: make check test test-student BASE_URL=$BASE_URL + - run: make test-observability BASE_URL=$BASE_URL + - run: make test-persistence BASE_URL=$BASE_URL ACTION_SCRIPT=scripts/restart.sh + - if: always() + run: docker compose down -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5e287bd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 MALLOC_ARENA_MAX=2 +WORKDIR /app +RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY src/ ./src/ +COPY migrations/ ./migrations/ +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] +STOPSIGNAL SIGTERM +CMD ["python", "src/server.py"] diff --git a/Makefile b/Makefile index 5d86d17..82e2a4b 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,19 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: test-student +test-student: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p 'test_*.py' -v + +.PHONY: test-observability test-incidents test-telemetry-outage +test-observability: + BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py proof +test-incidents: + BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py incidents +test-telemetry-outage: + BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py backend + +.PHONY: research +research: + BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability_cost.py diff --git a/REPORT.md b/REPORT.md index b97ae42..cf246d0 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,350 @@ -# Отчёт — лабораторная 6 +# Лабораторная 6 — наблюдаемость мессенджера -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Майорова Софья, БВТ2301. Целевая оценка: 5, трек 1 «Цена наблюдаемости».** +Дата выполнения: 20 ноября 2026 года. Оценку подтверждает преподаватель. + +Продолжена [собственная lab5, PR 7](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/7). +Перенесены только исходники, миграции, свои тесты и необходимые конфигурации. +README/COURSE/contracts/tests текущего skeleton lab6 сохранены, assertions публичных проверок не менялись. +Трек 2 с multi-window burn rate не заявляется. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Docker Engine/Compose v2, Linux-контейнеры, Python 3.9+ для host-проверок. +Сервер и worker Python 3.12.8 внутри Docker. На этом Mac: -## Архитектура и решения +```sh +colima start --profile mtusi-labs +export DOCKER_CONTEXT=colima-mtusi-labs +cd "/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab6" +python3 scripts/bootstrap.py +docker compose build api +docker compose up -d --wait +export BASE_URL=http://127.0.0.1:18093 +python3 scripts/wait_ready.py +make check test test-student BASE_URL=$BASE_URL +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +На другой машине клонировать публичный fork, выбрать solution/BVT2301-Mayorova и выполнять из корня clone. +Образ приложения сначала строится через build api: worker, dispatcher, migration и audit используют тот же image. +Bootstrap генерирует локальные случайные secrets, сохраняет существующие значения и .env. +Каталог secrets 0700; полный S3 identity config 0600, монтируется только root gateway. +Пароли, ключи, .env, volumes, backups и build не коммитятся и не попадают в build context. -## Доказательства критериев +API: http://127.0.0.1:18093. +Grafana: http://127.0.0.1:13006/d/messenger-lab6 — автоматический Viewer на localhost. +GF_USERS_VIEWERS_CAN_EDIT разрешает Explore и временные правки запросов без сохранения dashboard; +роль остаётся Viewer (настройка deprecated в Grafana 12.1.1, доступна в закреплённой версии). +Импорт dashboard и трёх datasources выполняет provisioning из Git; ручной настройки не требуется. +Prometheus http://127.0.0.1:19096, Loki 13106, Tempo 13206, собственные метрики Collector 18896. +OTLP 4318 не публикуется на host. PostgreSQL/S3/broker/API-реплики тоже не публикуются в обход LB. +Операторские UI присоединены к edge для корректного loopback port mapping Docker; backend и storage — внутренние сети. +Наружу в Интернет эти порты не открыты. Grafana не является административной панелью бизнес-данных. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Остановка с сохранением истории: docker compose down. +Не удалять volumes в проверке restart/failover. down -v предназначен только для намеренно уничтожаемого отдельного тестового стенда. +На этой машине старый Docker proxy не разрешался DNS. +После отдельного разрешения пользователя сохранён backup /etc/docker/daemon.json.lab6-proxy-backup внутри VM, +удалена неработающая proxy-секция и перезапущен Docker daemon. Реестр проверен по прямому HTTPS. +Это локальная настройка окружения, не часть приложения и не обязательный шаг для другого clone. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Архитектура -## Эксперименты и отказы +```text +Клиент → HAProxy → API ×2 → PostgreSQL + ├→ Redis cache профилей + └→ приватный S3 staging +PostgreSQL jobs/outbox → dispatcher → Redis Streams/AOF → worker → S3 results + DB ready -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +API / dispatcher / worker → OpenTelemetry SDK → OTLP HTTP/protobuf → Collector + ├→ metrics → Prometheus → Grafana + ├→ logs → Loki → Grafana Explore + └→ traces → Tempo → Grafana Explore +``` -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Prometheus хранит временные ряды, Loki — структурные логи, Tempo — трассы. +Grafana выполняет запросы и визуализирует; Collector принимает/обрабатывает/экспортирует и не заменяет долговременные хранилища. +PostgreSQL остаётся источником истины бизнес-данных и сессий. Наблюдаемость не участвует в COMMIT подтверждённого сообщения/задания. -## Ограничения и следующие шаги +Сохранён минимум предыдущих работ: scrypt и сессии с expiry/logout, Unicode/U+0000, ACL чатов, +keyset, атомарный Idempotency-Key на 24h, две реплики и migration job, Redis profile cache, +приватные изображения, durable staging/outbox, bounded renderer, at-least-once/retry/lease, +immutable versioned results, version fencing, failed registry/replay и операторский cleanup. +API возвращает 202 queued после S3 staging и COMMIT; resize выполняет отдельный worker. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Миграции 001–006. Новая 006 добавляет trace_context JSONB к attachment_jobs. +Миграция выполняется отдельным service под advisory lock; API проверяет наличие 006 до обслуживания. +Старые jobs получают пустой carrier и продолжают работать; бизнес-API не расширялся. + +## Версии и ресурсы + +Все Docker images закреплены version + digest в Compose/Dockerfile. +Python 3.12.8; PostgreSQL 17.2; Redis 7.4.1; SeaweedFS 4.51; HAProxy 3.0.5. +OTel Python API/SDK/HTTP exporter 1.36.0, semantic conventions 0.57b0; +Collector contrib 0.132.0; Prometheus 3.5.0; Loki 3.5.3; Tempo 2.8.2; Grafana 12.1.1. +Все 27 Python distributions, включая транзитивные, закреплены в requirements.txt по успешной сборке. + +API ×2: по 1 CPU/256MiB; worker 1 CPU/768MiB, concurrency1; dispatcher 1 CPU/256MiB. +PostgreSQL 1 CPU/512MiB; broker0.5CPU/192MiB/AOF always; Redis cache0.25CPU/64MiB. +S3 store1CPU/512MiB и gateway0.5CPU/512MiB; LB0.5CPU/64MiB. +Collector/Prometheus/Loki/Grafana — по 0.5CPU/256MiB, Tempo0.5CPU/512MiB. +Сумма CPU-лимитов — overcommit, не выделенные физические ядра. +Окружение: macOS arm64, Colima Linux VM4vCPU/около4GiB; локальный synthetic workload. + +## Схема сигналов и единицы + +Ручная SDK-инструментация ограничивает атрибуты разрешённым набором. +Resource: service.name=messenger-api / messenger-worker / messenger-dispatcher; +service.version=lab6.1; service.instance.id — случайный alias процесса; +deployment.environment.name=lab. Дополнительно SDK сообщает собственную версию/язык. + +| Реальное имя Prometheus | Значение | +| --- | --- | +| http_server_requests_total | Обработанные business requests; method, route template, status_class | +| http_server_duration_seconds_bucket/count/sum | Полная обработка HTTP, включая отправку ответа | +| jobs_processed_total | Попытки worker: ready, failed_or_retry, noop, fenced, error; job_type=image | +| job_processing_duration_seconds_bucket/count/sum | Время попытки, тот же outcome | +| job_ready_duration_seconds_bucket/count/sum | От created_at durable job перед COMMIT до публикации ready | +| job_retries_total | Решения retry/terminal failed, outcome=retry или failed | +| queue_depth | Незавершённые queued/processing в PostgreSQL, включая outbox до broker | +| queue_oldest_age_seconds | Возраст самой старой такой job; backlog не processing latency | +| queue_observation_ok | 0 при невозможности наблюдать PostgreSQL backlog | +| db_pool_used/waiting/capacity | Используемые/ожидающие/максимум соединений API | +| worker_busy/capacity | Активные попытки/число slots | +| process_cpu_seconds_total | CPU процесса, cumulative | +| process_resident_memory_bytes | RSS процесса через /proc/self/statm | +| container_memory_bytes | Cgroup memory.current, включая renderer и cache | +| spans_finished/exported/export_failed | Число sampled SDK spans и результат отправки | + +В gauge количества unit пустой: иначе unit1 экспортируется Prometheus как _ratio. +Очередь именуется постоянным attachment-jobs; отдельного динамического queue label не нужно. +Service instance — resource attribute с ограниченной жизнью процесса; churn реплик всё равно создаёт новые series. +Health полностью исключён из business metrics/logs/traces. Не распознанные маршруты имеют template unmatched. +user/chat/message/attachment/request/job/trace IDs, filename, raw URL и текст не являются labels метрик. + +Границы duration histogram, секунды: +0.001,0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2,5,10,20,30,60,+Inf. +Dashboard агрегирует buckets всех реплик: +histogram_quantile(0.95, sum by(le)(rate(http_server_duration_seconds_bucket[1m]))). +p50/p99 построены аналогично; средние p95 реплик не усредняются. +Точность percentile ограничена границами buckets и количеством наблюдений. + +## Трассы, логи и очередь + +Handler создаёт SERVER span с method/route/status без headers/body/query. +Connection context создаёт CLIENT db.transaction span, включающий ожидание пула, SQL и завершение транзакции, +но без текста SQL, параметров, connection string или сырого текста exception. +Это время unit of work, а не чистый server-side SQL execution time: S3 PUT под transaction виден дочерним S3 span. +S3 get span заканчивается при получении SDK response; потоковое чтение Body входит во время родительского HTTP/worker span. +S3 wrapper записывает только service S3 и имя операции; ключи, filenames, URLs и credentials не записываются. +Worker consumer span включает processing и реальные дочерние S3/DB spans. +Внутри изолированного renderer отдельный SDK не запускается. + +W3C traceparent/tracestate сохраняются с job в той же транзакции outbox. +Dispatcher извлекает durable context, создаёт PRODUCER span и вкладывает его carrier в envelope. +Worker извлекает envelope и создаёт CONSUMER child producer. Retry публикуется с тем же исходным trace, +новым producer/consumer и номером попытки. Envelope содержит ID/версию/попытку/carrier, а не bytes изображения. + +Структурный JSON log содержит timestamp, severity, service.name, event, trace_id, span_id, sampled; +HTTP — X-Request-Id, template/method/status/duration; job — технический job_id/attempt/outcome. +Эти технические ID нужны для сквозной диагностики синтетического задания и не индексируются как metric labels. +LoggingHandler отправляет OTLP logs; Loki принимает native OTLP и хранит остальные поля как structured metadata/body. +Grafana derived field TraceID ведёт из Loki в Tempo; tracesToLogs фильтрует тот же trace_id. +Имена полей/шаблоны фиксированы. Тела сообщений, картинки, пароли, Authorization, tokens и signed URLs исключены. + +Проверка proof выполняет реальный upload с известным traceparent, ждёт ready, +проверяет родителя consumer=producer и цепочку предков producer до SERVER upload, +проверяет worker S3/DB spans и логи всех стадий, связывает response X-Request-Id. +Отдельно остановлен worker, принят файл, приостановлен реальный S3 gateway, worker запущен: +наблюдается S3 timeout/retry; после возврата S3 второй consumer достигает ready в той же трассе. +Canary message/password/token/username/filename не найдены в проверяемом trace и app logs с начала controlled proof (лимит 5000); +это ограниченная динамическая проверка плюс allowlist кода, не математическое доказательство отсутствия любых утечек. +Метрики проверены на запрещённые ID labels. + +## SLI/SLO и алерты + +Полные определения/ограничения — infra/RUNBOOK.md. +Окно5мин: API availability>=99% обработанных business requests без5xx; health исключён, +4xx не смешиваются с отказами сервера. Отказ LB/сети до API данным SLI не покрывается. +Latency SLO: >=95% завершённых ready jobs укладываются в5сек от durable job creation. +Это условный latency SLO, не доступность pipeline; queued/failed отдельно видны через backlog/outcomes/alerts. + +Prometheus правила из infra/alerts.yaml, evaluation2s: +- APIHighErrorRate: 5xx share за30s>5%, for6s. +- QueueLag: oldest>10s и depth>0, for6s. +- QueueObservationUnavailable: observer0, for6s. +Это короткие учебные operational alerts, не трек2 multi-window burn rate. +Нет email/Slack или внешней доставки. + +## Два инцидента и диагностика + +1. Worker остановлен, API принял3 изображения. +Queue depth/oldest растут, завершения worker прекращаются; upload202 и логи acceptance/publish продолжаются. +QueueLag firing примерно через22.77сек. После возврата worker все три ready, backlog0 и alert resolved. +Значения gauges остановленного процесса могут оставаться в scrape cache до30s: это явно учитывается в runbook. + +2. Реальная ACCESS EXCLUSIVE блокировка sessions удерживается35s. +Readiness SELECT1 остаётся зелёной, защищённый endpoint ждёт SQL и получает503 по lock_timeout1.5s. +APIHighErrorRate firing примерно через13.14сек. В сохранённой трассе db.transaction — долгий/error span; +это отделяет ожидание PostgreSQL от HTTP handler/resize. После COMMIT блокировки прежний Bearer снова работает, +30s error window очищается и alert resolved. + +Сырые timestamps, firing snapshots, запросы метрик, error latencies и DB trace — evidence/incidents.json. +Разбор опирается на наблюдения; способ инъекции задокументирован отдельно и не заменяет trace/metric evidence. + +## Отказ telemetry backend и конечная буферизация + +Tempo остановлен примерно на56сек. За это время API подтвердил135 сообщений и5 файлов, +worker завершил файлы, все подтверждённые сообщения найдены в истории. +Collector memory snapshots: примерно58.21–62.05MiB при лимите256MiB; наблюдены156 failed exported spans. +После восстановления найден новый реальный trace. Потерянная telemetry не пересоздаётся из бизнес-данных. + +SDK: BatchSpanProcessor queue512/batch128, logs queue256/batch64, schedule500ms, +HTTP timeout1s; metric export каждые2s. Отправка асинхронная, очереди ограничены. +Collector memory_limiter limit192/spike32MiB, batch<=256; +два exporter queue по128 batches,2consumers, retry budget10s. +Prometheus exporter имеет expiration30s. Нет безразмерной очереди или бесконечного retry. +Лимит batches не является точным лимитом байт; SDK/receiver имеют дополнительные ограничения, +container memory лимит — последняя граница. Локальный тест конечной длительности не доказывает поведение при любом трафике. + +Метрики хранятся24h и ограничены256MB TSDB; Loki/Tempo retention24h; Grafana SQLite/provisioning. +Именованные metrics_data/logs_data/traces_data/grafana_data сохраняются при restart. +Loki/Tempo retention выполняется асинхронно и не является жёсткой дисковой квотой. + +## Исследование цены наблюдаемости + +Гипотеза: SDK export/sampling влияет на задержки, CPU/RAM и ingest; уменьшение доли трасс сокращает ingest, +но часть редких ошибок и медленных запросов не имеет полного trace. +Методика student-tests/observability_cost.py: +off, full1.0, head sampling0.10 и0.25; три повтора, порядок режимов циклически сдвигается. +Одинаковые лимиты,2API,worker concurrency1, fixture320x200. +Каждый прогон: фиксированные30 scheduled requests/s20s,600 запросов: +5% upload,15% messages,80% profile reads;16client threads. +Exporter off оставляет установленные SDK/hooks, но не создаёт providers и не экспортирует signals. +Это не сравнение с переписанной версией без SDK imports. + +Измеряются клиентские p50/p95/p99, подтверждённый RPS/ошибки, +CPU cgroup API/worker/dispatcher/Collector/хранилищ, +anonymous/working-set memory samples1Hz, +точные counter deltas принятых spans/logs/metric points и Collector receive network bytes. +Последние включают OTLP, Prometheus scrape и backend replies: это не чистый размер protobuf payload. +CPU/ingest включают ожидание exporter drain5s и проверки ready после основного потока. +История БД/telemetry сохраняется между режимами; новые API/worker/dispatcher создаются на каждом режиме. +Пять секунд прогрева/drain, фиксированная среда и смена порядка уменьшают, но не исключают шум/влияние кэша. +UUID/sampling стохастические; нагрузка локальная и короткая. Это не capacity ceiling и не production SLA. + +Первоначальные SQL-lock probes после измерительных окон не синхронизировали момент HTTP с pg_locks: +часть блокировок успевала завершиться при старте docker exec. Их фактические HTTP статусы сохранены +в rare_probes основного raw JSON; выдавать их за гарантированные 503 нельзя. +Для вывода о редких событиях выполнено отдельное исследование student-tests/sampling_retention.py: +full/10%/25%, три повтора; в каждом три настоящих503 и один медленный200. +Блокировки3.5s/1.2s проверяются через granted pg_locks перед HTTP, status и duration проверяются assertions. +У каждого режима получено9ошибок и3медленных успешных запроса. + +### Измеренные результаты + +Всего12основных прогонов,7200/7200 подтверждённых запросов без ошибок,360/360 изображений ready. +Ниже медиана трёх прогонов; ranges и каждый raw result доступны в JSON, точки/отрезки на графике. +CPU — накопленные секунды двух API или worker за измерительное окно с drain, не проценты host CPU. + +| Режим | HTTP p95, ms | p99, ms | API CPU, s | Worker CPU, s | Collector CPU, s | Принятые spans | Collector RX, KiB | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| off | 9.818 | 15.301 | 2.371 | 5.464 | 0.054 | 0 | 13.2 | +| full | 9.695 | 14.553 | 2.799 | 5.533 | 0.194 | 1950 | 1067.0 | +| 10% | 9.149 | 13.731 | 2.541 | 5.270 | 0.167 | 176 | 720.9 | +| 25% | 10.557 | 15.699 | 2.618 | 5.220 | 0.183 | 467 | 802.8 | + +| Режим | Сумма sampled anon peaks двух API, MiB | Collector anon peak, MiB | Logs / metric points, медиана | +| --- | ---: | ---: | ---: | +| off | 159.60 | 56.02 | 0 / 0 | +| full | 147.90 | 56.03 | 840 / 1011 | +| 10% | 168.66 | 58.78 | 840 / 987 | +| 25% | 147.82 | 58.72 | 840 / 1011 | + +Сумма отдельных sampled peaks не является одновременным peak всей группы; интервалы memory/p95 перекрываются. +Результаты не доказывают ускорение запросов или экономию RAM. Разница CPU небольшая и подвержена шуму. +Full даёт примерно18% больше API CPU, чем off, в медианах этого опыта. +25% уменьшает принятые spans на76.1%, сетевой RX Collector на24.8% и API CPU примерно на6.5% относительно full. +Ingest логов/метрик сохраняется независимо от trace sampling, поэтому суммарный трафик не падает на75%. +Head sampling применяется к целой цепочке: число дочерних spans в sampled image jobs меняет фактическую долю spans. +После drain sampled spans SDK и accepted spans Collector совпали во всех12прогонах: +дополнительная потеря между SDK и Collector в данном измерении0; несэмплированные spans в это число не входят. + +| Режим | Сохранён trace ошибок503 | Сохранён trace медленных200 | +| --- | ---: | ---: | +| full | 9/9 | 3/3 | +| 10% | 0/9 | 1/3 | +| 25% | 1/9 | 1/3 | + +Это наблюдения малой выборки, а не оценка вероятности или гарантия. Логи этих HTTP и полные метрики +сохраняются; sampled проверяется в коррелированном HTTP log; +для всех36probes также проверено реальное наличие sampled trace и404 для unsampled через Tempo API. Head sampling принимает решение до результата, +поэтому не может гарантировать trace ошибки. Для расследования воспроизводимого сбоя временно выбирать1.0 +или отправлять контролируемый sampled W3C parent (как proof); это не автоматическое сохранение всех ошибок. + +По умолчанию выбран25% как компромисс объёма трасс и покрытия при сохранённых logs/metrics. +10% даёт ещё меньший ingest, но более редкие полные причинные цепочки. В режиме incident диагностики подходит full. +Отдельный повтор выбранного режима:600/600 запросов,30/30 изображений ready, +p95=10.587ms,p99=17.222ms,456accepted spans, дополнительная потеря SDK→Collector0. +Результат — evidence/balanced-confirmation.json. +Сырые результаты: [observability-cost.json](evidence/observability-cost.json), +[cost-summary.json](evidence/cost-summary.json), [sampling-retention.json](evidence/sampling-retention.json). + +![Цена наблюдаемости, 3 повтора](evidence/observability-cost.png) + +## Проверки + +Локально выполнено: +- make check и полная OpenAPI3.1 validation — PASS, оригинальные contracts/tests без изменений. +- make test и make test-student — public smoke и18тестов API/изображений PASS. +- make test-observability — реальная causal trace, retry, correlated logs, low-cardinality и canary checks PASS. +- make test-incidents — оба алерта firing/resolved, trace SQL lock и восстановление PASS. +- make test-telemetry-outage — бизнес-записи/worker при отказе Tempo, bounded memory, новые traces после возврата PASS. +- make test-persistence ACTION_SCRIPT=scripts/restart.sh — confirmed business data и старый Bearer сохранены; + дополнительный persistence_telemetry.py подтвердил прежние spans/logs и historical Prometheus sample. +- make test-failover ACTION_SCRIPT=scripts/stop-one.sh —120записей,1временная ошибка, + первый успех после отказа2.602s; API восстановлен. +- student-tests/security.py — anonymous object/list403, strict envelope PASS. +- student-tests/pipeline_checks.py —13групп инвариантов, реальные exit77, lease recovery/version fencing/replay/cleanup PASS. +- make research —12прогонов; отдельные sampling_retention.py —9синхронизированных прогонов. +- promtool check rules —3правила PASS. +- Grafana UI: dashboard с реальными рядами, Tempo→Loki по тому же trace_id и Loki TraceID→Tempo проверены. + Скриншоты evidence/grafana.jpg, trace-ui.jpg, correlated-ui.jpg. + +Настоящий отдельный clone commit0e78b2b с новыми secrets и пустыми volumes: +сборка, миграции, public smoke,18тестов, causal/retry proof и полный restart с прежними +business data/Bearer/Tempo/Loki/Prometheus history — PASS за83.54s. +Тестовый Compose project mtusi-lab6-clean-6df218fa и только его volumes удалены; +основной mtusi-lab6 сохранён. Evidence: clean-clone.json / clean-clone.txt. +Изменения после clone ограничены отчётом, evidence и дополнительной проверкой исследования, бизнес-код не менялся. + +Команды для повторения fault/research проверок запускать последовательно на отдельном учебном стенде: +они управляют только текущим Compose project и временно останавливают его компоненты. +CI application workflow добавлен; результат hosted CI не подтверждён, требующее maintainer approval выполнение не выдаётся за PASS. +Сохранённые txt/JSON — фактические локальные результаты. Raw telemetry содержит только синтетические технические ID. + +## Ограничения + +Один host, один PostgreSQL/S3/broker/LB/Collector/backend каждого типа остаются SPOF. +Доказывается отказ API-процесса, перезапуск с сохранением volumes и ограниченный отказ telemetry backend, +а не потеря диска/питания, distributed HA или exactly-once. +Миграции и outbox определяют корректность данных независимо от наличия telemetry. +Логи и метрики тоже имеют bounded asynchronous exporters; при перегрузке/отказе Collector возможны их потери. +Head sampling не гарантирует сохранение ошибки или редкого хвоста; для этого потребовалась бы отдельная политика tail sampling +с доказанным буфером/памятью, которую мы не заявляем реализованной. +Retention24h и file-based stores рассчитаны на локальный synthetic стенд, не промышленное хранение. + +## Источники и использование ИИ + +Собственная lab5, skeleton/contracts/public tests lab6. +Официальные материалы: +- https://opentelemetry.io/docs/languages/python/instrumentation/ +- https://opentelemetry.io/docs/collector/configuration/ +- https://opentelemetry.io/docs/concepts/context-propagation/ +- https://grafana.com/docs/loki/latest/send-data/otel/ +- https://grafana.com/docs/grafana/latest/administration/provisioning/ +- https://prometheus.io/docs/practices/histograms/ + +Codex использован для реализации, проверки, controlled incidents, исследования и отчёта. +Численные результаты относятся к сохранённым локальным прогонам. Hosted CI не объявляется прошедшим без отдельного подтверждения. +\nНастройка Viewer/Explore: https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#viewers_can_edit\n \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..39a86ba --- /dev/null +++ b/compose.yaml @@ -0,0 +1,298 @@ +name: mtusi-lab6 +x-application: &application + image: mtusi-messenger:lab6 + networks: [backend] + environment: &environment + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + HTTP_WORKERS: ${HTTP_WORKERS:-16} + DB_POOL_MAX: ${DB_POOL_MAX:-8} + DB_ACCESS_MODE: ${DB_ACCESS_MODE:-optimized} + LOG_LEVEL: WARNING + TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1} + TRACE_SAMPLE_RATIO: ${TRACE_SAMPLE_RATIO:-0.25} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret + secrets: [database_url, cache_url, s3_api_key, s3_api_secret] + tmpfs: ["/tmp:size=128m,mode=1777"] + mem_limit: ${API_MEMORY:-512m} + cpus: ${API_CPUS:-1} + stop_grace_period: 10s +services: + migrate: + <<: *application + command: [python, src/migrate.py] + healthcheck: + disable: true + deploy: + replicas: 1 + depends_on: + postgres: + condition: service_healthy + api: + <<: *application + build: . + deploy: + replicas: ${API_REPLICAS:-2} + depends_on: + otel-collector: + condition: service_started + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + bucket-bootstrap: + condition: service_completed_successfully + bucket-bootstrap: + <<: *application + command: [python, src/bucket_bootstrap.py] + environment: + <<: *environment + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + secrets: [s3_admin_key, s3_admin_secret] + healthcheck: + disable: true + depends_on: + s3: + condition: service_healthy + audit: + <<: *application + profiles: [tools] + environment: + <<: *environment + BASE_URL: http://lb:8080 + BROKER_URL_FILE: /run/secrets/broker_url + JOB_LEASE_SECONDS: 5 + secrets: [database_url, cache_url, s3_api_key, s3_api_secret, s3_admin_key, s3_admin_secret, s3_worker_key, s3_worker_secret, broker_url] + healthcheck: + disable: true + deploy: + replicas: 1 + dispatcher: + <<: *application + command: [python, src/pipeline.py, dispatch] + environment: + <<: *environment + BROKER_URL_FILE: /run/secrets/broker_url + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60} + secrets: [database_url, broker_url] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + worker: + <<: *application + command: [python, src/pipeline.py, work] + environment: + <<: *environment + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + secrets: [database_url, broker_url, s3_worker_key, s3_worker_secret] + mem_limit: ${WORKER_MEMORY:-768m} + tmpfs: ["/tmp:size=128m,mode=1777"] + cpus: 1 + deploy: + replicas: 1 + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + bucket-bootstrap: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + broker: + image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47 + command: [redis-server, --appendonly, "yes", --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction, --save, ""] + volumes: [broker_data:/data] + networks: [backend] + mem_limit: 192m + cpus: 0.5 + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 30 + s3-store: + image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8 + command: [mini, -dir=/data, -ip=s3-store, -s3=false, -admin.ui=false, -webdav=false, -master.telemetry=false, -master.volumeSizeLimitMB=128, -volume.max=16, -volume.fileSizeLimitMB=11, -volume.concurrentUploadLimitMB=64, -volume.concurrentDownloadLimitMB=64] + networks: [storage] + volumes: [s3_data:/data] + mem_limit: 512m + cpus: 1 + stop_grace_period: 20s + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8888/] + interval: 2s + timeout: 2s + retries: 45 + s3: + image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8 + command: [s3, -filer=s3-store:8888, -port=8333, -config=/run/secrets/s3_config, -iam=false, -port.iceberg=0, -port.lance=0, -concurrentFileUploadLimit=16, -concurrentUploadLimitMB=32, -readerCacheSizeMB=16] + secrets: [s3_config] + networks: [backend, storage] + mem_limit: 512m + environment: + GOMEMLIMIT: 384MiB + cpus: 0.5 + depends_on: + s3-store: + condition: service_healthy + healthcheck: + test: [CMD-SHELL, 'nc -z 127.0.0.1 8333'] + interval: 2s + timeout: 1s + retries: 30 + lb: + image: haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949 + ports: ["127.0.0.1:${APP_PORT:-18093}:8080"] + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge, backend] + depends_on: + api: + condition: service_healthy + mem_limit: 64m + cpus: 0.5 + stop_grace_period: 12s + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 2s + timeout: 2s + retries: 10 + postgres: + image: postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + networks: [backend] + healthcheck: + test: [CMD, pg_isready, -h, 127.0.0.1, -U, messenger, -d, messenger] + interval: 2s + timeout: 1s + retries: 30 + mem_limit: 512m + cpus: ${POSTGRES_CPUS:-1} + redis: + image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47 + command: [redis-server, --maxmemory, 32mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + networks: [backend] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 30 + mem_limit: 64m + cpus: 0.25 + otel-collector: + image: otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af + command: [--config=/etc/otel/config.yaml] + volumes: [./infra/otel-collector.yaml:/etc/otel/config.yaml:ro] + networks: [backend, edge] + ports: ["127.0.0.1:${COLLECTOR_PORT:-18896}:8888"] + mem_limit: 256m + cpus: 0.5 + environment: + GOMEMLIMIT: 192MiB + prometheus: + image: prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996 + command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.path=/prometheus, --storage.tsdb.retention.time=24h, --storage.tsdb.retention.size=256MB] + volumes: [./infra/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro, ./infra/alerts.yaml:/etc/prometheus/alerts.yaml:ro, metrics_data:/prometheus] + networks: [backend, edge] + ports: ["127.0.0.1:${PROMETHEUS_PORT:-19096}:9090"] + mem_limit: 256m + cpus: 0.5 + loki: + image: grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd + command: [-config.file=/etc/loki/config.yaml] + volumes: [./infra/loki.yaml:/etc/loki/config.yaml:ro, logs_data:/loki] + networks: [backend, edge] + ports: ["127.0.0.1:${LOKI_PORT:-13106}:3100"] + mem_limit: 256m + cpus: 0.5 + environment: + GOMEMLIMIT: 192MiB + tempo: + image: grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7 + command: [-config.file=/etc/tempo/config.yaml] + volumes: [./infra/tempo.yaml:/etc/tempo/config.yaml:ro, traces_data:/var/tempo] + networks: [backend, edge] + ports: ["127.0.0.1:${TEMPO_PORT:-13206}:3200"] + mem_limit: 512m + cpus: 0.5 + environment: + GOMEMLIMIT: 384MiB + grafana: + image: grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d + volumes: [./infra/grafana/provisioning:/etc/grafana/provisioning:ro, ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro, grafana_data:/var/lib/grafana] + networks: [backend, edge] + ports: ["127.0.0.1:${GRAFANA_PORT:-13006}:3000"] + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_AUTH_DISABLE_LOGIN_FORM: "true" + GF_AUTH_BASIC_ENABLED: "false" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_USERS_VIEWERS_CAN_EDIT: "true" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + mem_limit: 256m + cpus: 0.5 + +volumes: + metrics_data: + logs_data: + traces_data: + grafana_data: + postgres_data: + broker_data: + s3_data: +networks: + edge: + storage: + internal: true + backend: + internal: true +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_admin_key: + file: ./secrets/s3_admin_key + s3_admin_secret: + file: ./secrets/s3_admin_secret + s3_api_key: + file: ./secrets/s3_api_key + s3_api_secret: + file: ./secrets/s3_api_secret + s3_config: + file: ./secrets/s3_config + broker_url: + file: ./secrets/broker_url + s3_worker_key: + file: ./secrets/s3_worker_key + s3_worker_secret: + file: ./secrets/s3_worker_secret diff --git a/evidence/backend-outage.json b/evidence/backend-outage.json new file mode 100644 index 0000000..6c0467a --- /dev/null +++ b/evidence/backend-outage.json @@ -0,0 +1,100 @@ +{ + "duration_seconds": 56.09580111503601, + "confirmed_messages": 135, + "confirmed_attachments_ready": 5, + "all_confirmed_messages_retained": true, + "collector_samples": [ + { + "elapsed": 2.1163909435272217, + "memory": "58.21MiB / 256MiB", + "cpu": "1.00%" + }, + { + "elapsed": 14.031727075576782, + "memory": "58.21MiB / 256MiB", + "cpu": "0.59%" + }, + { + "elapsed": 25.931473970413208, + "memory": "58.77MiB / 256MiB", + "cpu": "0.62%" + }, + { + "elapsed": 37.78569412231445, + "memory": "62.01MiB / 256MiB", + "cpu": "0.69%" + }, + { + "elapsed": 49.66645312309265, + "memory": "62.05MiB / 256MiB", + "cpu": "1.13%" + } + ], + "before": { + "otelcol_exporter_queue_capacity": 256.0, + "otelcol_exporter_queue_size": 0.0, + "otelcol_exporter_send_failed_log_records_total": 0.0, + "otelcol_exporter_send_failed_metric_points_total": 0.0, + "otelcol_exporter_send_failed_spans_total": 0.0, + "otelcol_exporter_sent_log_records_total": 824.0, + "otelcol_exporter_sent_metric_points_total": 16842.0, + "otelcol_exporter_sent_spans_total": 1783.0, + "otelcol_process_cpu_seconds_total": 2.12, + "otelcol_process_memory_rss_bytes": 203309056.0, + "otelcol_process_runtime_heap_alloc_bytes": 32249488.0, + "otelcol_process_runtime_total_alloc_bytes_total": 333861280.0, + "otelcol_process_runtime_total_sys_memory_bytes": 62215432.0, + "otelcol_process_uptime_seconds_total": 401.657121304, + "otelcol_processor_accepted_log_records_total": 824.0, + "otelcol_processor_accepted_metric_points_total": 16917.0, + "otelcol_processor_accepted_spans_total": 1783.0, + "otelcol_processor_batch_batch_send_size_bucket": 10026.0, + "otelcol_processor_batch_batch_send_size_sum": 19449.0, + "otelcol_processor_batch_batch_send_size_count": 478.0, + "otelcol_processor_batch_batch_size_trigger_send_total": 11.0, + "otelcol_processor_batch_metadata_cardinality": 3.0, + "otelcol_processor_batch_timeout_trigger_send_total": 467.0, + "otelcol_processor_incoming_items_total": 19524.0, + "otelcol_processor_outgoing_items_total": 19524.0, + "otelcol_receiver_accepted_log_records_total": 824.0, + "otelcol_receiver_accepted_metric_points_total": 16917.0, + "otelcol_receiver_accepted_spans_total": 1783.0, + "otelcol_receiver_refused_log_records_total": 0.0, + "otelcol_receiver_refused_metric_points_total": 0.0, + "otelcol_receiver_refused_spans_total": 0.0 + }, + "during": { + "otelcol_exporter_queue_capacity": 256.0, + "otelcol_exporter_queue_size": 37.0, + "otelcol_exporter_send_failed_log_records_total": 0.0, + "otelcol_exporter_send_failed_metric_points_total": 0.0, + "otelcol_exporter_send_failed_spans_total": 156.0, + "otelcol_exporter_sent_log_records_total": 1021.0, + "otelcol_exporter_sent_metric_points_total": 19388.0, + "otelcol_exporter_sent_spans_total": 1801.0, + "otelcol_process_cpu_seconds_total": 2.54, + "otelcol_process_memory_rss_bytes": 206245888.0, + "otelcol_process_runtime_heap_alloc_bytes": 41796440.0, + "otelcol_process_runtime_total_alloc_bytes_total": 385908760.0, + "otelcol_process_runtime_total_sys_memory_bytes": 66409736.0, + "otelcol_process_uptime_seconds_total": 456.688764525, + "otelcol_processor_accepted_log_records_total": 1021.0, + "otelcol_processor_accepted_metric_points_total": 19388.0, + "otelcol_processor_accepted_spans_total": 2356.0, + "otelcol_processor_batch_batch_send_size_bucket": 12845.0, + "otelcol_processor_batch_batch_send_size_sum": 22765.0, + "otelcol_processor_batch_batch_send_size_count": 606.0, + "otelcol_processor_batch_batch_size_trigger_send_total": 11.0, + "otelcol_processor_batch_metadata_cardinality": 3.0, + "otelcol_processor_batch_timeout_trigger_send_total": 595.0, + "otelcol_processor_incoming_items_total": 22765.0, + "otelcol_processor_outgoing_items_total": 22765.0, + "otelcol_receiver_accepted_log_records_total": 1021.0, + "otelcol_receiver_accepted_metric_points_total": 19388.0, + "otelcol_receiver_accepted_spans_total": 2356.0, + "otelcol_receiver_refused_log_records_total": 0.0, + "otelcol_receiver_refused_metric_points_total": 0.0, + "otelcol_receiver_refused_spans_total": 0.0 + }, + "recovered_trace_id": "1555228135a04659bac5654623c7a552" +} diff --git a/evidence/balanced-confirmation.json b/evidence/balanced-confirmation.json new file mode 100644 index 0000000..18e5b8b --- /dev/null +++ b/evidence/balanced-confirmation.json @@ -0,0 +1,141 @@ +{ + "mode": "balanced25", + "enabled": 1, + "sample_ratio": 0.25, + "repeat": 0, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.970493208, + "successful_rps": 30.044325583288316, + "p50_seconds": 0.003056625000001034, + "p95_seconds": 0.010586957999999314, + "p99_seconds": 0.017221916999998754, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.6645160000000008, + "peak_sampled_anonymous_bytes": 197382144, + "peak_sampled_working_set_bytes": 206495744, + "rx_bytes": 4570530 + }, + "broker": { + "cpu_seconds": 0.15844099999999983, + "peak_sampled_anonymous_bytes": 3289088, + "peak_sampled_working_set_bytes": 3899392, + "rx_bytes": 46110 + }, + "dispatcher": { + "cpu_seconds": 0.7753760000000005, + "peak_sampled_anonymous_bytes": 45830144, + "peak_sampled_working_set_bytes": 48340992, + "rx_bytes": 472693 + }, + "grafana": { + "cpu_seconds": 0.34897599999999995, + "peak_sampled_anonymous_bytes": 106962944, + "peak_sampled_working_set_bytes": 114311168, + "rx_bytes": 77173 + }, + "lb": { + "cpu_seconds": 0.24454800000000043, + "peak_sampled_anonymous_bytes": 6942720, + "peak_sampled_working_set_bytes": 7999488, + "rx_bytes": 4287798 + }, + "loki": { + "cpu_seconds": 0.16779699999999975, + "peak_sampled_anonymous_bytes": 108310528, + "peak_sampled_working_set_bytes": 112246784, + "rx_bytes": 111200 + }, + "otel-collector": { + "cpu_seconds": 0.19291599999999987, + "peak_sampled_anonymous_bytes": 57331712, + "peak_sampled_working_set_bytes": 63811584, + "rx_bytes": 863090 + }, + "postgres": { + "cpu_seconds": 1.631062, + "peak_sampled_anonymous_bytes": 15544320, + "peak_sampled_working_set_bytes": 38952960, + "rx_bytes": 1139116 + }, + "prometheus": { + "cpu_seconds": 0.1952210000000003, + "peak_sampled_anonymous_bytes": 56045568, + "peak_sampled_working_set_bytes": 59535360, + "rx_bytes": 116869 + }, + "redis": { + "cpu_seconds": 0.17422799999999983, + "peak_sampled_anonymous_bytes": 3067904, + "peak_sampled_working_set_bytes": 3612672, + "rx_bytes": 119742 + }, + "s3": { + "cpu_seconds": 0.4350659999999995, + "peak_sampled_anonymous_bytes": 69742592, + "peak_sampled_working_set_bytes": 71454720, + "rx_bytes": 11144479 + }, + "s3-store": { + "cpu_seconds": 0.9894250000000007, + "peak_sampled_anonymous_bytes": 134537216, + "peak_sampled_working_set_bytes": 137109504, + "rx_bytes": 7486398 + }, + "tempo": { + "cpu_seconds": 0.11411999999999978, + "peak_sampled_anonymous_bytes": 360697856, + "peak_sampled_working_set_bytes": 364331008, + "rx_bytes": 37382 + }, + "worker": { + "cpu_seconds": 5.430313999999999, + "peak_sampled_anonymous_bytes": 77266944, + "peak_sampled_working_set_bytes": 82108416, + "rx_bytes": 3761105 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 456.0, + "otelcol_receiver_accepted_log_records_total": 840.0, + "otelcol_receiver_accepted_metric_points_total": 1154.0, + "otelcol_exporter_sent_spans_total": 456.0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 456.0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "0c11495cc94443caadcd12bd17a86dd0", + "status": 503, + "latency_seconds": 1.4905514579999988, + "trace_sampled": true, + "trace_id": "aeacd5bdeb7d0f073b2a2cc44487c22c" + }, + { + "request_id": "b0871c2ae52245ce882338cf01bd6a4d", + "status": 503, + "latency_seconds": 1.4968927090000008, + "trace_sampled": false, + "trace_id": "4d8c492fae68617ec5dbd2bc849de2e9" + }, + { + "request_id": "d1d60c7ee15f498b988e031e15dc7d27", + "status": 503, + "latency_seconds": 1.4817318749999941, + "trace_sampled": false, + "trace_id": "931e742a91866246ac3b77dd062517c6" + }, + { + "request_id": "95e79f3e1894491c9355c15e0e79a695", + "status": 200, + "latency_seconds": 1.1869852499999993, + "trace_sampled": false, + "trace_id": "34e1ad7e9c366b666caf53045be238e9" + } + ] +} diff --git a/evidence/clean-clone.json b/evidence/clean-clone.json new file mode 100644 index 0000000..d2b5915 --- /dev/null +++ b/evidence/clean-clone.json @@ -0,0 +1,11 @@ +{ + "source_commit": "0e78b2b27de93464a6bd0974026bee4bfe4f602a", + "project": "mtusi-lab6-clean-6df218fa", + "new_secrets": true, + "new_empty_volumes": true, + "build_smoke_18_api_tests": true, + "real_observability_proof": true, + "business_and_telemetry_persistence": true, + "temporary_project_removed": true, + "seconds": 83.54 +} diff --git a/evidence/clean-clone.txt b/evidence/clean-clone.txt new file mode 100644 index 0000000..efb144e --- /dev/null +++ b/evidence/clean-clone.txt @@ -0,0 +1,338 @@ +RUN python3 scripts/bootstrap.py +Local configuration ready; no secrets printed. +RUN docker compose build api + Image mtusi-messenger:lab6 Building +#1 [internal] load local bake definitions +#1 reading from stdin 707B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 739B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#3 DONE 1.0s + +#4 [internal] load .dockerignore +#4 transferring context: 203B done +#4 DONE 0.0s + +#5 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#5 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done +#5 DONE 0.0s + +#6 [internal] load build context +#6 transferring context: 88.54kB done +#6 DONE 0.0s + +#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#7 CACHED + +#8 [5/7] RUN pip install --no-cache-dir -r requirements.txt +#8 CACHED + +#9 [6/7] COPY src/ ./src/ +#9 CACHED + +#10 [2/7] WORKDIR /app +#10 CACHED + +#11 [4/7] COPY requirements.txt ./ +#11 CACHED + +#12 [7/7] COPY migrations/ ./migrations/ +#12 CACHED + +#13 exporting to image +#13 exporting layers done +#13 exporting manifest sha256:e533c95082fcc2d03df488d24c04c74c3205f96ebce7d39fcf544e15ed2ac2c9 done +#13 exporting config sha256:d5159a690cc9eed35b1e1df5510f9f8595fbb5e303c09bde83ea92197bbfab8c done +#13 exporting attestation manifest sha256:91fc899a62ccb18479e1135cbfe8c85ae865353f7315c7c8897a6418fe283c11 done +#13 exporting manifest list sha256:a0c792a12285d3f4914e1ce37a8832777d639abd2653a71ff0b581b28f2f5e6b done +#13 naming to docker.io/library/mtusi-messenger:lab6 done +#13 unpacking to docker.io/library/mtusi-messenger:lab6 done +#13 DONE 0.0s + +#14 resolving provenance for metadata file +#14 DONE 0.0s + Image mtusi-messenger:lab6 Built +RUN docker compose up -d --wait --wait-timeout 180 + Network mtusi-lab6-clean-6df218fa_edge Creating + Network mtusi-lab6-clean-6df218fa_edge Created + Network mtusi-lab6-clean-6df218fa_backend Creating + Network mtusi-lab6-clean-6df218fa_backend Created + Network mtusi-lab6-clean-6df218fa_storage Creating + Network mtusi-lab6-clean-6df218fa_storage Created + Volume mtusi-lab6-clean-6df218fa_logs_data Creating + Volume mtusi-lab6-clean-6df218fa_logs_data Created + Volume mtusi-lab6-clean-6df218fa_metrics_data Creating + Volume mtusi-lab6-clean-6df218fa_metrics_data Created + Volume mtusi-lab6-clean-6df218fa_traces_data Creating + Volume mtusi-lab6-clean-6df218fa_traces_data Created + Volume mtusi-lab6-clean-6df218fa_postgres_data Creating + Volume mtusi-lab6-clean-6df218fa_postgres_data Created + Volume mtusi-lab6-clean-6df218fa_s3_data Creating + Volume mtusi-lab6-clean-6df218fa_s3_data Created + Volume mtusi-lab6-clean-6df218fa_broker_data Creating + Volume mtusi-lab6-clean-6df218fa_broker_data Created + Volume mtusi-lab6-clean-6df218fa_grafana_data Creating + Volume mtusi-lab6-clean-6df218fa_grafana_data Created + Container mtusi-lab6-clean-6df218fa-grafana-1 Creating + Container mtusi-lab6-clean-6df218fa-s3-store-1 Creating + Container mtusi-lab6-clean-6df218fa-prometheus-1 Creating + Container mtusi-lab6-clean-6df218fa-otel-collector-1 Creating + Container mtusi-lab6-clean-6df218fa-loki-1 Creating + Container mtusi-lab6-clean-6df218fa-redis-1 Creating + Container mtusi-lab6-clean-6df218fa-postgres-1 Creating + Container mtusi-lab6-clean-6df218fa-broker-1 Creating + Container mtusi-lab6-clean-6df218fa-tempo-1 Creating + Container mtusi-lab6-clean-6df218fa-otel-collector-1 Created + Container mtusi-lab6-clean-6df218fa-grafana-1 Created + Container mtusi-lab6-clean-6df218fa-prometheus-1 Created + Container mtusi-lab6-clean-6df218fa-loki-1 Created + Container mtusi-lab6-clean-6df218fa-tempo-1 Created + Container mtusi-lab6-clean-6df218fa-s3-store-1 Created + Container mtusi-lab6-clean-6df218fa-broker-1 Created + Container mtusi-lab6-clean-6df218fa-s3-1 Creating + Container mtusi-lab6-clean-6df218fa-redis-1 Created + Container mtusi-lab6-clean-6df218fa-postgres-1 Created + Container mtusi-lab6-clean-6df218fa-migrate-1 Creating + Container mtusi-lab6-clean-6df218fa-migrate-1 Created + Container mtusi-lab6-clean-6df218fa-dispatcher-1 Creating + Container mtusi-lab6-clean-6df218fa-s3-1 Created + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Creating + Container mtusi-lab6-clean-6df218fa-dispatcher-1 Created + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Created + Container mtusi-lab6-clean-6df218fa-worker-1 Creating + Container mtusi-lab6-clean-6df218fa-api-1 Creating + Container mtusi-lab6-clean-6df218fa-api-2 Creating + Container mtusi-lab6-clean-6df218fa-api-1 Created + Container mtusi-lab6-clean-6df218fa-api-2 Created + Container mtusi-lab6-clean-6df218fa-lb-1 Creating + Container mtusi-lab6-clean-6df218fa-worker-1 Created + Container mtusi-lab6-clean-6df218fa-lb-1 Created + Container mtusi-lab6-clean-6df218fa-s3-store-1 Starting + Container mtusi-lab6-clean-6df218fa-redis-1 Starting + Container mtusi-lab6-clean-6df218fa-tempo-1 Starting + Container mtusi-lab6-clean-6df218fa-prometheus-1 Starting + Container mtusi-lab6-clean-6df218fa-loki-1 Starting + Container mtusi-lab6-clean-6df218fa-broker-1 Starting + Container mtusi-lab6-clean-6df218fa-grafana-1 Starting + Container mtusi-lab6-clean-6df218fa-otel-collector-1 Starting + Container mtusi-lab6-clean-6df218fa-postgres-1 Starting + Container mtusi-lab6-clean-6df218fa-s3-store-1 Started + Container mtusi-lab6-clean-6df218fa-s3-store-1 Waiting + Container mtusi-lab6-clean-6df218fa-broker-1 Started + Container mtusi-lab6-clean-6df218fa-redis-1 Started + Container mtusi-lab6-clean-6df218fa-otel-collector-1 Started + Container mtusi-lab6-clean-6df218fa-prometheus-1 Started + Container mtusi-lab6-clean-6df218fa-loki-1 Started + Container mtusi-lab6-clean-6df218fa-tempo-1 Started + Container mtusi-lab6-clean-6df218fa-postgres-1 Started + Container mtusi-lab6-clean-6df218fa-postgres-1 Waiting + Container mtusi-lab6-clean-6df218fa-grafana-1 Started + Container mtusi-lab6-clean-6df218fa-postgres-1 Healthy + Container mtusi-lab6-clean-6df218fa-migrate-1 Starting + Container mtusi-lab6-clean-6df218fa-migrate-1 Started + Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting + Container mtusi-lab6-clean-6df218fa-broker-1 Waiting + Container mtusi-lab6-clean-6df218fa-broker-1 Healthy + Container mtusi-lab6-clean-6df218fa-migrate-1 Exited + Container mtusi-lab6-clean-6df218fa-dispatcher-1 Starting + Container mtusi-lab6-clean-6df218fa-dispatcher-1 Started + Container mtusi-lab6-clean-6df218fa-s3-store-1 Healthy + Container mtusi-lab6-clean-6df218fa-s3-1 Starting + Container mtusi-lab6-clean-6df218fa-s3-1 Started + Container mtusi-lab6-clean-6df218fa-s3-1 Waiting + Container mtusi-lab6-clean-6df218fa-s3-1 Healthy + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Starting + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Started + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting + Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting + Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting + Container mtusi-lab6-clean-6df218fa-redis-1 Waiting + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting + Container mtusi-lab6-clean-6df218fa-broker-1 Waiting + Container mtusi-lab6-clean-6df218fa-broker-1 Healthy + Container mtusi-lab6-clean-6df218fa-redis-1 Healthy + Container mtusi-lab6-clean-6df218fa-migrate-1 Exited + Container mtusi-lab6-clean-6df218fa-migrate-1 Exited + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited + Container mtusi-lab6-clean-6df218fa-worker-1 Starting + Container mtusi-lab6-clean-6df218fa-api-1 Starting + Container mtusi-lab6-clean-6df218fa-worker-1 Started + Container mtusi-lab6-clean-6df218fa-api-1 Started + Container mtusi-lab6-clean-6df218fa-api-2 Starting + Container mtusi-lab6-clean-6df218fa-api-2 Started + Container mtusi-lab6-clean-6df218fa-api-1 Waiting + Container mtusi-lab6-clean-6df218fa-api-2 Waiting + Container mtusi-lab6-clean-6df218fa-api-1 Healthy + Container mtusi-lab6-clean-6df218fa-api-2 Healthy + Container mtusi-lab6-clean-6df218fa-lb-1 Starting + Container mtusi-lab6-clean-6df218fa-lb-1 Started + Container mtusi-lab6-clean-6df218fa-broker-1 Waiting + Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting + Container mtusi-lab6-clean-6df218fa-worker-1 Waiting + Container mtusi-lab6-clean-6df218fa-prometheus-1 Waiting + Container mtusi-lab6-clean-6df218fa-lb-1 Waiting + Container mtusi-lab6-clean-6df218fa-loki-1 Waiting + Container mtusi-lab6-clean-6df218fa-tempo-1 Waiting + Container mtusi-lab6-clean-6df218fa-dispatcher-1 Waiting + Container mtusi-lab6-clean-6df218fa-otel-collector-1 Waiting + Container mtusi-lab6-clean-6df218fa-grafana-1 Waiting + Container mtusi-lab6-clean-6df218fa-redis-1 Waiting + Container mtusi-lab6-clean-6df218fa-s3-store-1 Waiting + Container mtusi-lab6-clean-6df218fa-postgres-1 Waiting + Container mtusi-lab6-clean-6df218fa-s3-1 Waiting + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting + Container mtusi-lab6-clean-6df218fa-api-1 Waiting + Container mtusi-lab6-clean-6df218fa-api-2 Waiting + Container mtusi-lab6-clean-6df218fa-s3-store-1 Healthy + Container mtusi-lab6-clean-6df218fa-loki-1 Healthy + Container mtusi-lab6-clean-6df218fa-migrate-1 Exited + Container mtusi-lab6-clean-6df218fa-broker-1 Healthy + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited + Container mtusi-lab6-clean-6df218fa-postgres-1 Healthy + Container mtusi-lab6-clean-6df218fa-worker-1 Healthy + Container mtusi-lab6-clean-6df218fa-tempo-1 Healthy + Container mtusi-lab6-clean-6df218fa-otel-collector-1 Healthy + Container mtusi-lab6-clean-6df218fa-grafana-1 Healthy + Container mtusi-lab6-clean-6df218fa-redis-1 Healthy + Container mtusi-lab6-clean-6df218fa-dispatcher-1 Healthy + Container mtusi-lab6-clean-6df218fa-s3-1 Healthy + Container mtusi-lab6-clean-6df218fa-prometheus-1 Healthy + Container mtusi-lab6-clean-6df218fa-api-1 Healthy + Container mtusi-lab6-clean-6df218fa-api-2 Healthy + Container mtusi-lab6-clean-6df218fa-lb-1 Healthy +RUN python3 scripts/wait_ready.py +Ready through load balancer. +RUN make check test test-student BASE_URL=http://127.0.0.1:18094 +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:18094" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +BASE_URL="http://127.0.0.1:18094" python3 -m unittest discover -s student-tests -p 'test_*.py' -v +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok +test_async_variants_and_replay (test_images.Images) ... ok +test_mime_bytes_pixels_and_filename (test_images.Images) ... ok +test_private_and_cross_chat (test_images.Images) ... ok + +---------------------------------------------------------------------- +Ran 18 tests in 4.950s + +OK +RUN python3 -u student-tests/observability.py proof + Container mtusi-lab6-clean-6df218fa-worker-1 Stopping + Container mtusi-lab6-clean-6df218fa-worker-1 Stopped + Container mtusi-lab6-clean-6df218fa-s3-1 Paused + Container mtusi-lab6-clean-6df218fa-worker-1 Starting + Container mtusi-lab6-clean-6df218fa-worker-1 Started + Container mtusi-lab6-clean-6df218fa-s3-1 Unpaused + Container mtusi-lab6-clean-6df218fa-worker-1 Running +PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality. +RUN python3 -u student-tests/persistence_telemetry.py +PASS: full business restart, original Bearer, historical Tempo trace, Loki logs and fixed-time Prometheus sample. +RUN docker compose down -v --remove-orphans + Container mtusi-lab6-clean-6df218fa-loki-1 Stopping + Container mtusi-lab6-clean-6df218fa-lb-1 Stopping + Container mtusi-lab6-clean-6df218fa-grafana-1 Stopping + Container mtusi-lab6-clean-6df218fa-tempo-1 Stopping + Container mtusi-lab6-clean-6df218fa-dispatcher-1 Stopping + Container mtusi-lab6-clean-6df218fa-prometheus-1 Stopping + Container mtusi-lab6-clean-6df218fa-worker-1 Stopping + Container mtusi-lab6-clean-6df218fa-dispatcher-1 Stopped + Container mtusi-lab6-clean-6df218fa-dispatcher-1 Removing + Container mtusi-lab6-clean-6df218fa-dispatcher-1 Removed + Container mtusi-lab6-clean-6df218fa-lb-1 Stopped + Container mtusi-lab6-clean-6df218fa-lb-1 Removing + Container mtusi-lab6-clean-6df218fa-lb-1 Removed + Container mtusi-lab6-clean-6df218fa-api-2 Stopping + Container mtusi-lab6-clean-6df218fa-api-1 Stopping + Container mtusi-lab6-clean-6df218fa-worker-1 Stopped + Container mtusi-lab6-clean-6df218fa-worker-1 Removing + Container mtusi-lab6-clean-6df218fa-worker-1 Removed + Container mtusi-lab6-clean-6df218fa-broker-1 Stopping + Container mtusi-lab6-clean-6df218fa-grafana-1 Stopped + Container mtusi-lab6-clean-6df218fa-grafana-1 Removing + Container mtusi-lab6-clean-6df218fa-prometheus-1 Stopped + Container mtusi-lab6-clean-6df218fa-prometheus-1 Removing + Container mtusi-lab6-clean-6df218fa-grafana-1 Removed + Container mtusi-lab6-clean-6df218fa-prometheus-1 Removed + Container mtusi-lab6-clean-6df218fa-broker-1 Stopped + Container mtusi-lab6-clean-6df218fa-broker-1 Removing + Container mtusi-lab6-clean-6df218fa-broker-1 Removed + Container mtusi-lab6-clean-6df218fa-api-1 Stopped + Container mtusi-lab6-clean-6df218fa-api-1 Removing + Container mtusi-lab6-clean-6df218fa-api-1 Removed + Container mtusi-lab6-clean-6df218fa-api-2 Stopped + Container mtusi-lab6-clean-6df218fa-api-2 Removing + Container mtusi-lab6-clean-6df218fa-api-2 Removed + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Stopping + Container mtusi-lab6-clean-6df218fa-migrate-1 Stopping + Container mtusi-lab6-clean-6df218fa-otel-collector-1 Stopping + Container mtusi-lab6-clean-6df218fa-redis-1 Stopping + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Stopped + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Removing + Container mtusi-lab6-clean-6df218fa-migrate-1 Stopped + Container mtusi-lab6-clean-6df218fa-migrate-1 Removing + Container mtusi-lab6-clean-6df218fa-migrate-1 Removed + Container mtusi-lab6-clean-6df218fa-postgres-1 Stopping + Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Removed + Container mtusi-lab6-clean-6df218fa-s3-1 Stopping + Container mtusi-lab6-clean-6df218fa-redis-1 Stopped + Container mtusi-lab6-clean-6df218fa-redis-1 Removing + Container mtusi-lab6-clean-6df218fa-postgres-1 Stopped + Container mtusi-lab6-clean-6df218fa-postgres-1 Removing + Container mtusi-lab6-clean-6df218fa-redis-1 Removed + Container mtusi-lab6-clean-6df218fa-postgres-1 Removed + Container mtusi-lab6-clean-6df218fa-loki-1 Stopped + Container mtusi-lab6-clean-6df218fa-loki-1 Removing + Container mtusi-lab6-clean-6df218fa-loki-1 Removed + Container mtusi-lab6-clean-6df218fa-s3-1 Stopped + Container mtusi-lab6-clean-6df218fa-s3-1 Removing + Container mtusi-lab6-clean-6df218fa-otel-collector-1 Stopped + Container mtusi-lab6-clean-6df218fa-otel-collector-1 Removing + Container mtusi-lab6-clean-6df218fa-s3-1 Removed + Container mtusi-lab6-clean-6df218fa-s3-store-1 Stopping + Container mtusi-lab6-clean-6df218fa-otel-collector-1 Removed + Container mtusi-lab6-clean-6df218fa-s3-store-1 Stopped + Container mtusi-lab6-clean-6df218fa-s3-store-1 Removing + Container mtusi-lab6-clean-6df218fa-s3-store-1 Removed + Container mtusi-lab6-clean-6df218fa-tempo-1 Stopped + Container mtusi-lab6-clean-6df218fa-tempo-1 Removing + Container mtusi-lab6-clean-6df218fa-tempo-1 Removed + Volume mtusi-lab6-clean-6df218fa_broker_data Removing + Network mtusi-lab6-clean-6df218fa_storage Removing + Volume mtusi-lab6-clean-6df218fa_metrics_data Removing + Volume mtusi-lab6-clean-6df218fa_grafana_data Removing + Volume mtusi-lab6-clean-6df218fa_traces_data Removing + Network mtusi-lab6-clean-6df218fa_edge Removing + Volume mtusi-lab6-clean-6df218fa_s3_data Removing + Volume mtusi-lab6-clean-6df218fa_logs_data Removing + Volume mtusi-lab6-clean-6df218fa_postgres_data Removing + Network mtusi-lab6-clean-6df218fa_backend Removing + Volume mtusi-lab6-clean-6df218fa_broker_data Removed + Volume mtusi-lab6-clean-6df218fa_metrics_data Removed + Volume mtusi-lab6-clean-6df218fa_logs_data Removed + Volume mtusi-lab6-clean-6df218fa_grafana_data Removed + Volume mtusi-lab6-clean-6df218fa_traces_data Removed + Volume mtusi-lab6-clean-6df218fa_s3_data Removed + Volume mtusi-lab6-clean-6df218fa_postgres_data Removed + Network mtusi-lab6-clean-6df218fa_storage Removed + Network mtusi-lab6-clean-6df218fa_backend Removed + Network mtusi-lab6-clean-6df218fa_edge Removed diff --git a/evidence/correlated-ui.jpg b/evidence/correlated-ui.jpg new file mode 100644 index 0000000..5b6097e Binary files /dev/null and b/evidence/correlated-ui.jpg differ diff --git a/evidence/cost-summary.json b/evidence/cost-summary.json new file mode 100644 index 0000000..4ac5bab --- /dev/null +++ b/evidence/cost-summary.json @@ -0,0 +1,86 @@ +{ + "off": { + "runs": 3, + "requests": 1800, + "ready_images": 90, + "p95_ms": 9.81791700002077, + "p99_ms": 15.300541999977213, + "api_cpu_seconds": 2.3706709999999998, + "worker_cpu_seconds": 5.463901, + "collector_cpu_seconds": 0.053601000000000454, + "api_sum_sampled_anon_peak_mib": 159.59765625, + "collector_anon_peak_mib": 56.015625, + "accepted_spans": 0, + "accepted_logs": 0, + "accepted_metric_points": 0, + "collector_receive_kib": 13.2080078125, + "sampled_spans_not_accepted": 0, + "error_probe_traces_kept": 0, + "error_probes": 1, + "slow_probe_traces_kept": 0, + "slow_probes": 11 + }, + "full": { + "runs": 3, + "requests": 1800, + "ready_images": 90, + "p95_ms": 9.695250000049782, + "p99_ms": 14.552875000049426, + "api_cpu_seconds": 2.7994320000000004, + "worker_cpu_seconds": 5.533063, + "collector_cpu_seconds": 0.19410300000000014, + "api_sum_sampled_anon_peak_mib": 147.90234375, + "collector_anon_peak_mib": 56.02734375, + "accepted_spans": 1950.0, + "accepted_logs": 840.0, + "accepted_metric_points": 1011.0, + "collector_receive_kib": 1067.025390625, + "sampled_spans_not_accepted": 0, + "error_probe_traces_kept": 4, + "error_probes": 4, + "slow_probe_traces_kept": 8, + "slow_probes": 8 + }, + "sample10": { + "runs": 3, + "requests": 1800, + "ready_images": 90, + "p95_ms": 9.149458999957005, + "p99_ms": 13.731374999991885, + "api_cpu_seconds": 2.540737, + "worker_cpu_seconds": 5.2699810000000005, + "collector_cpu_seconds": 0.16662400000000055, + "api_sum_sampled_anon_peak_mib": 168.65625, + "collector_anon_peak_mib": 58.78125, + "accepted_spans": 176.0, + "accepted_logs": 840.0, + "accepted_metric_points": 987.0, + "collector_receive_kib": 720.869140625, + "sampled_spans_not_accepted": 0, + "error_probe_traces_kept": 0, + "error_probes": 1, + "slow_probe_traces_kept": 0, + "slow_probes": 11 + }, + "sample25": { + "runs": 3, + "requests": 1800, + "ready_images": 90, + "p95_ms": 10.556667000003017, + "p99_ms": 15.699083000015435, + "api_cpu_seconds": 2.6178429999999997, + "worker_cpu_seconds": 5.2202720000000005, + "collector_cpu_seconds": 0.18281499999999973, + "api_sum_sampled_anon_peak_mib": 147.81640625, + "collector_anon_peak_mib": 58.71875, + "accepted_spans": 467.0, + "accepted_logs": 840.0, + "accepted_metric_points": 1011.0, + "collector_receive_kib": 802.8349609375, + "sampled_spans_not_accepted": 0, + "error_probe_traces_kept": 1, + "error_probes": 2, + "slow_probe_traces_kept": 3, + "slow_probes": 10 + } +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..c537b6a --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18093" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 1, "first_recovery_seconds": 2.602} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/final-api-tests.txt b/evidence/final-api-tests.txt new file mode 100644 index 0000000..cc6138f --- /dev/null +++ b/evidence/final-api-tests.txt @@ -0,0 +1,28 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:18093" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +BASE_URL="http://127.0.0.1:18093" python3 -m unittest discover -s student-tests -p 'test_*.py' -v +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok +test_async_variants_and_replay (test_images.Images) ... ok +test_mime_bytes_pixels_and_filename (test_images.Images) ... ok +test_private_and_cross_chat (test_images.Images) ... ok + +---------------------------------------------------------------------- +Ran 18 tests in 5.414s + +OK diff --git a/evidence/final-validation.txt b/evidence/final-validation.txt new file mode 100644 index 0000000..3bc31b6 --- /dev/null +++ b/evidence/final-validation.txt @@ -0,0 +1 @@ +PASS: Python syntax; original README/COURSE/CONTRIBUTING/contracts/public tests unchanged; actual local PostgreSQL password and S3 credentials absent from tracked/new files. Non-secret internal Redis service URLs intentionally appear in bootstrap. diff --git a/evidence/grafana.jpg b/evidence/grafana.jpg new file mode 100644 index 0000000..983104e Binary files /dev/null and b/evidence/grafana.jpg differ diff --git a/evidence/incidents.json b/evidence/incidents.json new file mode 100644 index 0000000..f7adf34 --- /dev/null +++ b/evidence/incidents.json @@ -0,0 +1,7683 @@ +{ + "worker_stop": { + "injected_at": 1795163117.556709, + "detected_at": 1795163140.329111, + "detection_seconds": 22.772402048110962, + "firing": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/RUNBOOK.md#queue-lag", + "summary": "Accepted unfinished image work older than 10 seconds" + }, + "state": "firing", + "activeAt": "2026-11-20T08:25:32.478517494Z", + "value": "1.7749819e+01" + } + ], + "metrics": { + "queue_depth": [ + { + "metric": { + "__name__": "queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "11a51b423246479e8ca2b43c8ec27a38", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "11a51b423246479e8ca2b43c8ec27a38", + "service_name": "messenger-api", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.428, + "0" + ] + }, + { + "metric": { + "__name__": "queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "8233dde3da6d4f13ab70ce40e99fc803", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "8233dde3da6d4f13ab70ce40e99fc803", + "service_name": "messenger-api", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.428, + "0" + ] + }, + { + "metric": { + "__name__": "queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "fb68c5cfe0a64787bba2c8036179590b", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "fb68c5cfe0a64787bba2c8036179590b", + "service_name": "messenger-dispatcher", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.428, + "3" + ] + }, + { + "metric": { + "__name__": "queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "c16c37ace1824e80bfe39a6f30807466", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "c16c37ace1824e80bfe39a6f30807466", + "service_name": "messenger-worker", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.428, + "0" + ] + } + ], + "queue_oldest_age_seconds": [ + { + "metric": { + "__name__": "queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "11a51b423246479e8ca2b43c8ec27a38", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "11a51b423246479e8ca2b43c8ec27a38", + "service_name": "messenger-api", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.43, + "0" + ] + }, + { + "metric": { + "__name__": "queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "8233dde3da6d4f13ab70ce40e99fc803", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "8233dde3da6d4f13ab70ce40e99fc803", + "service_name": "messenger-api", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.43, + "0" + ] + }, + { + "metric": { + "__name__": "queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "fb68c5cfe0a64787bba2c8036179590b", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "fb68c5cfe0a64787bba2c8036179590b", + "service_name": "messenger-dispatcher", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.43, + "19.929928" + ] + }, + { + "metric": { + "__name__": "queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "c16c37ace1824e80bfe39a6f30807466", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "c16c37ace1824e80bfe39a6f30807466", + "service_name": "messenger-worker", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.43, + "0" + ] + } + ], + "worker_busy": [ + { + "metric": { + "__name__": "worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "11a51b423246479e8ca2b43c8ec27a38", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "11a51b423246479e8ca2b43c8ec27a38", + "service_name": "messenger-api", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.432, + "0" + ] + }, + { + "metric": { + "__name__": "worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "8233dde3da6d4f13ab70ce40e99fc803", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "8233dde3da6d4f13ab70ce40e99fc803", + "service_name": "messenger-api", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.432, + "0" + ] + }, + { + "metric": { + "__name__": "worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "fb68c5cfe0a64787bba2c8036179590b", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "fb68c5cfe0a64787bba2c8036179590b", + "service_name": "messenger-dispatcher", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.432, + "0" + ] + }, + { + "metric": { + "__name__": "worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "c16c37ace1824e80bfe39a6f30807466", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "c16c37ace1824e80bfe39a6f30807466", + "service_name": "messenger-worker", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.432, + "0" + ] + } + ], + "worker_capacity": [ + { + "metric": { + "__name__": "worker_capacity", + "deployment_environment_name": "lab", + "exported_instance": "11a51b423246479e8ca2b43c8ec27a38", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "11a51b423246479e8ca2b43c8ec27a38", + "service_name": "messenger-api", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.434, + "0" + ] + }, + { + "metric": { + "__name__": "worker_capacity", + "deployment_environment_name": "lab", + "exported_instance": "8233dde3da6d4f13ab70ce40e99fc803", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "8233dde3da6d4f13ab70ce40e99fc803", + "service_name": "messenger-api", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.434, + "0" + ] + }, + { + "metric": { + "__name__": "worker_capacity", + "deployment_environment_name": "lab", + "exported_instance": "fb68c5cfe0a64787bba2c8036179590b", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "fb68c5cfe0a64787bba2c8036179590b", + "service_name": "messenger-dispatcher", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.434, + "0" + ] + }, + { + "metric": { + "__name__": "worker_capacity", + "deployment_environment_name": "lab", + "exported_instance": "c16c37ace1824e80bfe39a6f30807466", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "c16c37ace1824e80bfe39a6f30807466", + "service_name": "messenger-worker", + "service_version": "lab6.1", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.36.0" + }, + "value": [ + 1795163140.434, + "1" + ] + } + ] + }, + "diagnosis": "Backlog and oldest age grow; worker completion stops while API acceptance still succeeds. Gauges expire after 30s.", + "resolved_at": 1795163146.0027971 + }, + "db_lock": { + "injected_at": 1795163147.011447, + "detected_at": 1795163160.154182, + "detection_seconds": 13.142735004425049, + "firing": [ + { + "labels": { + "alertname": "APIHighErrorRate", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/RUNBOOK.md#api-errors", + "summary": "API 5xx share exceeds 5 percent in the last 30 seconds" + }, + "state": "firing", + "activeAt": "2026-11-20T08:25:52.478517494Z", + "value": "7.484207931500946e-01" + } + ], + "error_latencies_seconds": [ + 1.510284459000001, + 1.5103016249999968, + 1.511464666000002, + 1.5110016249999987, + 1.51131775, + 1.511445166999998, + 1.5126569580000009, + 1.5140159170000018, + 1.6104141659999982, + 1.6125100829999965, + 1.6111711249999985, + 1.612879999999997, + 1.4968940829999937, + 1.4961242500000012, + 1.4942949580000047, + 1.4961574999999954, + 1.4933542079999995, + 1.4932850000000002, + 1.495215666, + 1.4941746249999994, + 1.4944899589999991, + 1.4968713330000014, + 1.4982131249999995, + 1.4975980829999997, + 1.4968774170000003, + 1.496618583, + 1.4952612920000021, + 1.4951001660000003, + 1.4972887499999956, + 1.498597457999999, + 1.500068290999998, + 1.4998819999999995 + ], + "trace_id": "285ae9502aea442f99fe2dea27e72719", + "resolved_at": 1795163214.1534898, + "trace": { + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "8233dde3da6d4f13ab70ce40e99fc803" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "VaC2lZS0Jvk=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163147116546319", + "endTimeUnixNano": "1795163148619210223", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "zUOzMbzkwSk=", + "parentSpanId": "VaC2lZS0Jvk=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163147116618248", + "endTimeUnixNano": "1795163148618934756", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "VxiULXWaVJc=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163147116896045", + "endTimeUnixNano": "1795163148621553865", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "/+uJIGxCNXA=", + "parentSpanId": "VxiULXWaVJc=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163147116920508", + "endTimeUnixNano": "1795163148621330741", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "TLPZKH3DSqs=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163148628117246", + "endTimeUnixNano": "1795163150132835625", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "m+DXABYxyNw=", + "parentSpanId": "TLPZKH3DSqs=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163148628161379", + "endTimeUnixNano": "1795163150132398699", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "G3XjWBCNerg=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163148628411592", + "endTimeUnixNano": "1795163150134092770", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "d7j3PMuT/aE=", + "parentSpanId": "G3XjWBCNerg=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163148628444723", + "endTimeUnixNano": "1795163150131851998", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "uV4yHdysDKQ=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163150142550622", + "endTimeUnixNano": "1795163151645721387", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "GBDmlm8bziU=", + "parentSpanId": "uV4yHdysDKQ=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163150142615928", + "endTimeUnixNano": "1795163151645045841", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "/Q6sXHdTcTE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163150142938164", + "endTimeUnixNano": "1795163151647007530", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "jaTeKj+sb1o=", + "parentSpanId": "/Q6sXHdTcTE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163150142998343", + "endTimeUnixNano": "1795163151646582150", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "KZJVoAIlGGU=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163151653128255", + "endTimeUnixNano": "1795163153154988969", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "84Nns23Ooec=", + "parentSpanId": "KZJVoAIlGGU=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163151653189744", + "endTimeUnixNano": "1795163153154749353", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "eX0QwcdYFfE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163151653486976", + "endTimeUnixNano": "1795163153155467867", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "+rjZSFJPjzs=", + "parentSpanId": "eX0QwcdYFfE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163151653523911", + "endTimeUnixNano": "1795163153155262631", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "CAhIv+rchD8=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163153160722437", + "endTimeUnixNano": "1795163154663952272", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "lBd5omItFkw=", + "parentSpanId": "CAhIv+rchD8=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163153160766329", + "endTimeUnixNano": "1795163154663703857", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "uBBg5deuv4w=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163153161067975", + "endTimeUnixNano": "1795163154663117119", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ltBl+PMf5nQ=", + "parentSpanId": "uBBg5deuv4w=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163153161102651", + "endTimeUnixNano": "1795163154662849957", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "0RfCGv+5opQ=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163154668711205", + "endTimeUnixNano": "1795163156172276912", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "5R+lhVoUp80=", + "parentSpanId": "0RfCGv+5opQ=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163154668784006", + "endTimeUnixNano": "1795163156171530502", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Obv9Lbcf+QQ=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163154669339010", + "endTimeUnixNano": "1795163156171999316", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "2gaVEc6zXlM=", + "parentSpanId": "Obv9Lbcf+QQ=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163154669377680", + "endTimeUnixNano": "1795163156171106241", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "K7T8XtqnpEc=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163156178533132", + "endTimeUnixNano": "1795163157679739803", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "h1wRzuLcC6M=", + "parentSpanId": "K7T8XtqnpEc=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163156178600507", + "endTimeUnixNano": "1795163157679589733", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "U/Swk/plQJM=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163156179402924", + "endTimeUnixNano": "1795163157680688638", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ByzrCGZ3xno=", + "parentSpanId": "U/Swk/plQJM=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163156179465768", + "endTimeUnixNano": "1795163157680526664", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "QjpLfF9h9RE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163157682826505", + "endTimeUnixNano": "1795163159185277429", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "0vt9Bfevdb0=", + "parentSpanId": "QjpLfF9h9RE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163157682857777", + "endTimeUnixNano": "1795163159184788868", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "7X6+kN9Hcyg=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163157683025536", + "endTimeUnixNano": "1795163159185080874", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "z6V1ZLHKWHU=", + "parentSpanId": "7X6+kN9Hcyg=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163157683049262", + "endTimeUnixNano": "1795163159184465756", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "gIVhvsjLejQ=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163159190320535", + "endTimeUnixNano": "1795163160692623117", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "utQQuKrmAkM=", + "parentSpanId": "gIVhvsjLejQ=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163159190379564", + "endTimeUnixNano": "1795163160692297666", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "oadQZHFBDEM=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163159190708704", + "endTimeUnixNano": "1795163160693097779", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "FBScL6hLNhA=", + "parentSpanId": "oadQZHFBDEM=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163159190739266", + "endTimeUnixNano": "1795163160692919823", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ffAvuytvwcI=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163160699063565", + "endTimeUnixNano": "1795163162200470938", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "541iDkbPMag=", + "parentSpanId": "ffAvuytvwcI=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163160699119969", + "endTimeUnixNano": "1795163162200276498", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "l1s2jr4xd6c=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163160699344538", + "endTimeUnixNano": "1795163162200862872", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "u61nBmDUuWA=", + "parentSpanId": "l1s2jr4xd6c=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163160699373116", + "endTimeUnixNano": "1795163162200594141", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Jw0r03caAqM=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163162204180989", + "endTimeUnixNano": "1795163163707287503", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "9T+DnzEEnlQ=", + "parentSpanId": "Jw0r03caAqM=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163162204213598", + "endTimeUnixNano": "1795163163706605760", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "okR0pK9XAic=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163162204389560", + "endTimeUnixNano": "1795163163707587439", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "zCXB6WmCdz8=", + "parentSpanId": "okR0pK9XAic=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163162204420079", + "endTimeUnixNano": "1795163163706936037", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "/uPydmBGI9A=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163163716355521", + "endTimeUnixNano": "1795163165218659555", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "uV5Xbqch0Xw=", + "parentSpanId": "/uPydmBGI9A=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163163716417121", + "endTimeUnixNano": "1795163165218011289", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "EZV60sQ25NE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163163716718980", + "endTimeUnixNano": "1795163165218493531", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "oIhvADa82+g=", + "parentSpanId": "EZV60sQ25NE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163163716749154", + "endTimeUnixNano": "1795163165218296223", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "tazSwamC+2Q=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163165223303130", + "endTimeUnixNano": "1795163166724879672", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "tpwcedn/5es=", + "parentSpanId": "tazSwamC+2Q=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163165223349700", + "endTimeUnixNano": "1795163166724633221", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "y+AXzhlx2TY=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163165223584932", + "endTimeUnixNano": "1795163166725147334", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "RgDvDOe70aw=", + "parentSpanId": "y+AXzhlx2TY=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163165223615171", + "endTimeUnixNano": "1795163166725001101", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "/B3RW5F+4AE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163166730454786", + "endTimeUnixNano": "1795163168232138885", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "1uI2z9HnQoc=", + "parentSpanId": "/B3RW5F+4AE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163166730497587", + "endTimeUnixNano": "1795163168231806253", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "7vllanbVt5A=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163166730680984", + "endTimeUnixNano": "1795163168232235515", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "L423MDX/F/Q=", + "parentSpanId": "7vllanbVt5A=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163166730706998", + "endTimeUnixNano": "1795163168231966830", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "L/7z6lnUTA4=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163168236537361", + "endTimeUnixNano": "1795163169739248849", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "whd6oosb16s=", + "parentSpanId": "L/7z6lnUTA4=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163168236568751", + "endTimeUnixNano": "1795163169739076988", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "zwHGq8ezcSk=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163168236749071", + "endTimeUnixNano": "1795163169738797369", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "4R0KZmwEHGs=", + "parentSpanId": "zwHGq8ezcSk=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163168236782088", + "endTimeUnixNano": "1795163169738389711", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "cAScj+5RgkA=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163169743482650", + "endTimeUnixNano": "1795163171246023020", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Bk9wfFOc1/w=", + "parentSpanId": "cAScj+5RgkA=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163169743544125", + "endTimeUnixNano": "1795163171245745436", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "5EUy7II0gEg=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163169743943520", + "endTimeUnixNano": "1795163171245570115", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "dZfKIhcXc70=", + "parentSpanId": "5EUy7II0gEg=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163169743975989", + "endTimeUnixNano": "1795163171245351903", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "11a51b423246479e8ca2b43c8ec27a38" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "BDRJrbE6SA0=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163147116751354", + "endTimeUnixNano": "1795163148619812210", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Zf0ZGgLnqEM=", + "parentSpanId": "BDRJrbE6SA0=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163147116793986", + "endTimeUnixNano": "1795163148619561080", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "75P7jUjiO+E=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163147117082827", + "endTimeUnixNano": "1795163148619250772", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "GM0H1VSz+GM=", + "parentSpanId": "75P7jUjiO+E=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163147117108623", + "endTimeUnixNano": "1795163148618934547", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "zgn0I4R4isY=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163148627842570", + "endTimeUnixNano": "1795163150132489343", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "knON0o968Bc=", + "parentSpanId": "zgn0I4R4isY=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163148627903540", + "endTimeUnixNano": "1795163150131960605", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "6ERFwDPLSrA=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163148628250062", + "endTimeUnixNano": "1795163150133759407", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Vuw8sbYo5mM=", + "parentSpanId": "6ERFwDPLSrA=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163148628282777", + "endTimeUnixNano": "1795163150133344526", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ytVakbTdNEc=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163150142497694", + "endTimeUnixNano": "1795163151645583985", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "JyyaR/QX3jY=", + "parentSpanId": "ytVakbTdNEc=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163150142574586", + "endTimeUnixNano": "1795163151645053887", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "QSPpTwI14Rg=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163150143140832", + "endTimeUnixNano": "1795163151647004570", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "s7BknUiRfhU=", + "parentSpanId": "QSPpTwI14Rg=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163150143190926", + "endTimeUnixNano": "1795163151646570978", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "USz+ZFsjZao=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163151653066891", + "endTimeUnixNano": "1795163153154940491", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ZPmJk9O47v8=", + "parentSpanId": "USz+ZFsjZao=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163151653132007", + "endTimeUnixNano": "1795163153154543486", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "p63FdFi9qnw=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163151653634175", + "endTimeUnixNano": "1795163153155243652", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ELRTN99lsB0=", + "parentSpanId": "p63FdFi9qnw=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163151653673236", + "endTimeUnixNano": "1795163153155073976", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "RA6J/jK3DHE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163153160525954", + "endTimeUnixNano": "1795163154663138934", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "rdmV3g7Yw8A=", + "parentSpanId": "RA6J/jK3DHE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163153160578220", + "endTimeUnixNano": "1795163154662886862", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "m+ggRsTt1PY=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163153161023241", + "endTimeUnixNano": "1795163154663586753", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "OBOEz6f7XXY=", + "parentSpanId": "m+ggRsTt1PY=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163153161056570", + "endTimeUnixNano": "1795163154663381086", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "49kd22Nd3BU=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163154669275666", + "endTimeUnixNano": "1795163156172159279", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Y02pof0crRs=", + "parentSpanId": "49kd22Nd3BU=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163154669327409", + "endTimeUnixNano": "1795163156171741688", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "1C2mwUzqARY=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163154669596000", + "endTimeUnixNano": "1795163156172737336", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "1HDaup7+OA4=", + "parentSpanId": "1C2mwUzqARY=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163154669631770", + "endTimeUnixNano": "1795163156171316631", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ZSvo1EzKFlw=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163156177934309", + "endTimeUnixNano": "1795163157679858559", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "jGflidvTwfQ=", + "parentSpanId": "ZSvo1EzKFlw=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163156178007767", + "endTimeUnixNano": "1795163157679646869", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "BAG5TggdLOQ=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163156178566862", + "endTimeUnixNano": "1795163157680101019", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "F7FnsNWhlRY=", + "parentSpanId": "BAG5TggdLOQ=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163156178620309", + "endTimeUnixNano": "1795163157679467707", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Yt7/hO+J66U=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163157682822103", + "endTimeUnixNano": "1795163159185237490", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "N36mce4Wan0=", + "parentSpanId": "Yt7/hO+J66U=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163157682856142", + "endTimeUnixNano": "1795163159184475134", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "+si6mr+ngzE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163157683033123", + "endTimeUnixNano": "1795163159185104653", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "rQ4DjzObh9Y=", + "parentSpanId": "+si6mr+ngzE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163157683056640", + "endTimeUnixNano": "1795163159184913750", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "bKdF98QCkGU=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163159190521988", + "endTimeUnixNano": "1795163160692551441", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "TG/TJtrvwOk=", + "parentSpanId": "bKdF98QCkGU=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163159190579970", + "endTimeUnixNano": "1795163160692259882", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "dcls1LanVus=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163159190878298", + "endTimeUnixNano": "1795163160693748891", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "C5cKSjMfAR4=", + "parentSpanId": "dcls1LanVus=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163159190907394", + "endTimeUnixNano": "1795163160693418879", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "3R0W8dtF24g=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163160699036744", + "endTimeUnixNano": "1795163162200836325", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ziZ4du1Hozc=", + "parentSpanId": "3R0W8dtF24g=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163160699106495", + "endTimeUnixNano": "1795163162200281096", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "3pwZ/Phoso4=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163160699331357", + "endTimeUnixNano": "1795163162200646148", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "n7WYa2uioB4=", + "parentSpanId": "3pwZ/Phoso4=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163160699359434", + "endTimeUnixNano": "1795163162200457519", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "K9UmWB6QUw0=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163162204080278", + "endTimeUnixNano": "1795163163707753644", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "m1Ub8qmcn9Y=", + "parentSpanId": "K9UmWB6QUw0=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163162204115479", + "endTimeUnixNano": "1795163163707466119", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "k+HagEPAMik=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163162204600514", + "endTimeUnixNano": "1795163163706938879", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "MbiVLAoTnvg=", + "parentSpanId": "k+HagEPAMik=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163162204623800", + "endTimeUnixNano": "1795163163706594602", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "NKxiZYWty2w=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163163716382225", + "endTimeUnixNano": "1795163165218653457", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "a5VxaWzynsA=", + "parentSpanId": "NKxiZYWty2w=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163163716436763", + "endTimeUnixNano": "1795163165218015632", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "MQOoIopv91o=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163163716710538", + "endTimeUnixNano": "1795163165218466633", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "WP8lJKUvq48=", + "parentSpanId": "MQOoIopv91o=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163163716746813", + "endTimeUnixNano": "1795163165218263144", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "y4cHvrN34BE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163165223106783", + "endTimeUnixNano": "1795163166725023274", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "2qIAxqcyejU=", + "parentSpanId": "y4cHvrN34BE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163165223161038", + "endTimeUnixNano": "1795163166724932369", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "VBjvchwTDA0=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163165223446390", + "endTimeUnixNano": "1795163166724831275", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "7hsE/emI1oc=", + "parentSpanId": "VBjvchwTDA0=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163165223471325", + "endTimeUnixNano": "1795163166724635017", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "MkEaC8VBx1U=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163166730450151", + "endTimeUnixNano": "1795163168232101193", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "szCJHhhPuhs=", + "parentSpanId": "MkEaC8VBx1U=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163166730498589", + "endTimeUnixNano": "1795163168231962113", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "TdA4Lsrqgf8=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163166730704785", + "endTimeUnixNano": "1795163168232203291", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "2GROAXT4irw=", + "parentSpanId": "TdA4Lsrqgf8=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163166730728629", + "endTimeUnixNano": "1795163168231791268", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Ch3eXSTSzTc=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163168236517827", + "endTimeUnixNano": "1795163169739179863", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "QpflcQreyrM=", + "parentSpanId": "Ch3eXSTSzTc=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163168236552973", + "endTimeUnixNano": "1795163169738989180", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ht56Tnnr8wE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163168236737300", + "endTimeUnixNano": "1795163169738833010", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "cwpCci+9+F0=", + "parentSpanId": "ht56Tnnr8wE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163168236763346", + "endTimeUnixNano": "1795163169738461744", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "3AqsChofLYI=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163169743485697", + "endTimeUnixNano": "1795163171245995399", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ig/ZF+T7RHw=", + "parentSpanId": "3AqsChofLYI=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163169743543666", + "endTimeUnixNano": "1795163171245840606", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "6VQ3SkD8rjY=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163169743971649", + "endTimeUnixNano": "1795163171245664409", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "8vvjRAIAA/Q=", + "parentSpanId": "6VQ3SkD8rjY=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163169744003116", + "endTimeUnixNano": "1795163171245343266", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "8233dde3da6d4f13ab70ce40e99fc803" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "xVSbaWHLtgs=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163171251241799", + "endTimeUnixNano": "1795163172753298069", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "9lvt9C2OVa8=", + "parentSpanId": "xVSbaWHLtgs=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163171251289155", + "endTimeUnixNano": "1795163172752576001", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "3FVikXMEig0=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163171251628615", + "endTimeUnixNano": "1795163172753054307", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "0wQRQmvaJsU=", + "parentSpanId": "3FVikXMEig0=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163171251649560", + "endTimeUnixNano": "1795163172752885888", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "9cjvGElVZmM=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163172757701857", + "endTimeUnixNano": "1795163174259607406", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "L4eG8aeQJTI=", + "parentSpanId": "9cjvGElVZmM=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163172757738945", + "endTimeUnixNano": "1795163174258882420", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "+ey/l71B8Bk=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163172757880788", + "endTimeUnixNano": "1795163174259439977", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Na2Wnq8up4c=", + "parentSpanId": "+ey/l71B8Bk=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163172757896892", + "endTimeUnixNano": "1795163174259173441", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "tiySnmwn6JE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163174264893827", + "endTimeUnixNano": "1795163175768909530", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "XQoFGd02FiE=", + "parentSpanId": "tiySnmwn6JE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163174264963444", + "endTimeUnixNano": "1795163175768519140", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "mO2gcZ5yR5Q=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163174265288584", + "endTimeUnixNano": "1795163175768073277", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "pUWy9CPFoAk=", + "parentSpanId": "mO2gcZ5yR5Q=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163174265316072", + "endTimeUnixNano": "1795163175767268888", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "uJNFgzM7dy4=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163175774989451", + "endTimeUnixNano": "1795163177277887215", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "W5ssEjJ3cxE=", + "parentSpanId": "uJNFgzM7dy4=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163175775065902", + "endTimeUnixNano": "1795163177277496036", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "jlbOiTYw2wI=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163175775340927", + "endTimeUnixNano": "1795163177277206030", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ycveYyipuLg=", + "parentSpanId": "jlbOiTYw2wI=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163175775378173", + "endTimeUnixNano": "1795163177276792540", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "O1/JPc+SrbA=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163177284299715", + "endTimeUnixNano": "1795163178787210196", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "25VANoU3fpE=", + "parentSpanId": "O1/JPc+SrbA=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163177284351428", + "endTimeUnixNano": "1795163178786171870", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "tyIYqKMoFFg=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163177284939197", + "endTimeUnixNano": "1795163178786965575", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "nuBRzH39M2k=", + "parentSpanId": "tyIYqKMoFFg=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163177284977397", + "endTimeUnixNano": "1795163178786584006", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "bzDfwmRJ+0k=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163178794085901", + "endTimeUnixNano": "1795163180297653821", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Z4w4EtjZis0=", + "parentSpanId": "bzDfwmRJ+0k=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163178794167177", + "endTimeUnixNano": "1795163180297360557", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "HIJS4+1e+yo=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163178794624851", + "endTimeUnixNano": "1795163180296623499", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "0u/NFNRwzO0=", + "parentSpanId": "HIJS4+1e+yo=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163178794668513", + "endTimeUnixNano": "1795163180296152143", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "6IFhWj2pwbY=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163180305097998", + "endTimeUnixNano": "1795163181294372214", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ZM6pRnincpo=", + "parentSpanId": "6IFhWj2pwbY=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163180305164548", + "endTimeUnixNano": "1795163181294033719", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "2qZZsbaV3Gw=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163180305454726", + "endTimeUnixNano": "1795163181293877564", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "WiXkJHtLaxA=", + "parentSpanId": "2qZZsbaV3Gw=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163180305497800", + "endTimeUnixNano": "1795163181293346012", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "U2B6DzKKC94=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181297328565", + "endTimeUnixNano": "1795163181298991687", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "wDkegWjbgxg=", + "parentSpanId": "U2B6DzKKC94=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181297372013", + "endTimeUnixNano": "1795163181298641225", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ATOyh+llU4A=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181298285759", + "endTimeUnixNano": "1795163181299375465", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "alHlYpamIyI=", + "parentSpanId": "ATOyh+llU4A=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181298316865", + "endTimeUnixNano": "1795163181299216100", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "D4koG1Vqk/Q=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181301127444", + "endTimeUnixNano": "1795163181302284032", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "eZahb5gVFvw=", + "parentSpanId": "D4koG1Vqk/Q=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181301163303", + "endTimeUnixNano": "1795163181301977393", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "0mFtpvbKjfo=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181301322335", + "endTimeUnixNano": "1795163181302130546", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "OqPg+jEBfCE=", + "parentSpanId": "0mFtpvbKjfo=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181301344851", + "endTimeUnixNano": "1795163181301886285", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "9fqZEe+0Fzk=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181304861734", + "endTimeUnixNano": "1795163181306034834", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "B8/6zJoAvTo=", + "parentSpanId": "9fqZEe+0Fzk=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181304893090", + "endTimeUnixNano": "1795163181305939473", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ArY2pFy8t0o=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181305067925", + "endTimeUnixNano": "1795163181305894107", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "i8HQSHdjMps=", + "parentSpanId": "ArY2pFy8t0o=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181305090024", + "endTimeUnixNano": "1795163181305634836", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "iO+BfylFenY=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181308352639", + "endTimeUnixNano": "1795163181309185785", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "b88r/EHiB1w=", + "parentSpanId": "iO+BfylFenY=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181308384412", + "endTimeUnixNano": "1795163181309016287", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "TThB/0i247Q=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181308590270", + "endTimeUnixNano": "1795163181309389432", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "vMwXFjKOVFs=", + "parentSpanId": "TThB/0i247Q=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181308610159", + "endTimeUnixNano": "1795163181309281437", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "75dfyEEPxks=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181311871231", + "endTimeUnixNano": "1795163181313064888", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "T0/ovocfQpQ=", + "parentSpanId": "75dfyEEPxks=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181311910718", + "endTimeUnixNano": "1795163181312933835", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "P1j8GCOBF0U=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181312120203", + "endTimeUnixNano": "1795163181312858280", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "0OPBCmKqtoM=", + "parentSpanId": "P1j8GCOBF0U=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181312143262", + "endTimeUnixNano": "1795163181312647252", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "2VHF9u711+Y=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181315995179", + "endTimeUnixNano": "1795163181317215688", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "QNEke99SvRU=", + "parentSpanId": "2VHF9u711+Y=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181316029412", + "endTimeUnixNano": "1795163181317109778", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "9Y2Z14h4sow=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181316179646", + "endTimeUnixNano": "1795163181317042688", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "iIF4XKnplhA=", + "parentSpanId": "9Y2Z14h4sow=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181316195741", + "endTimeUnixNano": "1795163181316831326", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "DjY7D2sWZqA=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181319442010", + "endTimeUnixNano": "1795163181321084993", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "DwPCNOr0NG0=", + "parentSpanId": "DjY7D2sWZqA=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181319468780", + "endTimeUnixNano": "1795163181320948477", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "1fqw8UyCUnA=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181319833002", + "endTimeUnixNano": "1795163181320872380", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "28DJaDb3EfU=", + "parentSpanId": "1fqw8UyCUnA=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181319852725", + "endTimeUnixNano": "1795163181320547853", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ivNXyxhOHHY=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181323409970", + "endTimeUnixNano": "1795163181324448556", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "QicN5PRLXBQ=", + "parentSpanId": "ivNXyxhOHHY=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181323436989", + "endTimeUnixNano": "1795163181324357573", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "I+8egs3Sw78=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181323592769", + "endTimeUnixNano": "1795163181324280934", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "CEKW+Jhb0Vs=", + "parentSpanId": "I+8egs3Sw78=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181323613200", + "endTimeUnixNano": "1795163181324106516", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "KXbie6whVy4=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181326592818", + "endTimeUnixNano": "1795163181327744653", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "bXFh0zdHDnI=", + "parentSpanId": "KXbie6whVy4=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181326619004", + "endTimeUnixNano": "1795163181327660968", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "+VweUCXyLzE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181326835619", + "endTimeUnixNano": "1795163181327613141", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "9qKh12703AE=", + "parentSpanId": "+VweUCXyLzE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181326854758", + "endTimeUnixNano": "1795163181327406992", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "11a51b423246479e8ca2b43c8ec27a38" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "L7mrMbjWV08=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163171251513333", + "endTimeUnixNano": "1795163172753451928", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "A8qJZLpasrI=", + "parentSpanId": "L7mrMbjWV08=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163171251553930", + "endTimeUnixNano": "1795163172752744920", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "qKNMOg5kd+8=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163171251791961", + "endTimeUnixNano": "1795163172753224561", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "MVaS7EBy3Ac=", + "parentSpanId": "qKNMOg5kd+8=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163171251817579", + "endTimeUnixNano": "1795163172752577420", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "RZ3mfP06Y5U=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163172757318879", + "endTimeUnixNano": "1795163174259415367", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "SkTiSLGumWQ=", + "parentSpanId": "RZ3mfP06Y5U=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163172757359680", + "endTimeUnixNano": "1795163174258886675", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "/8xDew9Vpx8=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163172757603692", + "endTimeUnixNano": "1795163174259721988", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "JgKYI64vbPY=", + "parentSpanId": "/8xDew9Vpx8=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163172757626721", + "endTimeUnixNano": "1795163174259154212", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "SW/GOektqFs=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163174265096044", + "endTimeUnixNano": "1795163175768890970", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "VGPUS1x3g8s=", + "parentSpanId": "SW/GOektqFs=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163174265155358", + "endTimeUnixNano": "1795163175767256125", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "87LWzY58A24=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163174265424521", + "endTimeUnixNano": "1795163175769248453", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "xwQDy//+TdA=", + "parentSpanId": "87LWzY58A24=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163174265449632", + "endTimeUnixNano": "1795163175768260798", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "cin2LeQ9t/Q=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163175775019147", + "endTimeUnixNano": "1795163177277130797", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "tNd7iff40Co=", + "parentSpanId": "cin2LeQ9t/Q=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163175775084421", + "endTimeUnixNano": "1795163177276844878", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "DMnrK3fBh/c=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163175775347225", + "endTimeUnixNano": "1795163177277724196", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "i30xWvk797w=", + "parentSpanId": "DMnrK3fBh/c=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163175775383220", + "endTimeUnixNano": "1795163177277393988", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "aOJANJqIvYI=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163177284057543", + "endTimeUnixNano": "1795163178787509697", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "+LYbAN3MqfE=", + "parentSpanId": "aOJANJqIvYI=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163177284114552", + "endTimeUnixNano": "1795163178786513030", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "z67d+K2I468=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163177284437796", + "endTimeUnixNano": "1795163178786967702", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "1ubq9gKFdgE=", + "parentSpanId": "z67d+K2I468=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163177284464236", + "endTimeUnixNano": "1795163178786099727", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "R8pPe9/QjXw=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163178794386902", + "endTimeUnixNano": "1795163180296690508", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ntd8tkHvq+o=", + "parentSpanId": "R8pPe9/QjXw=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163178794458545", + "endTimeUnixNano": "1795163180296201430", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "lDKbKydQIj8=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163178794804376", + "endTimeUnixNano": "1795163180297720455", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "GB15iM+Uj+0=", + "parentSpanId": "lDKbKydQIj8=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163178794851374", + "endTimeUnixNano": "1795163180297439617", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "LockNotAvailable" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "M4UYlvZLxj8=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163180305122725", + "endTimeUnixNano": "1795163181294059487", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "57S7wnM3HC0=", + "parentSpanId": "M4UYlvZLxj8=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163180305181728", + "endTimeUnixNano": "1795163181293842914", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "muByvK56/kQ=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163180305464358", + "endTimeUnixNano": "1795163181293227342", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "frBgqp33XsI=", + "parentSpanId": "muByvK56/kQ=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163180305507808", + "endTimeUnixNano": "1795163181292810874", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Yn5/F2QPMIw=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181297216776", + "endTimeUnixNano": "1795163181298595609", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "B6QRDa/Glw4=", + "parentSpanId": "Yn5/F2QPMIw=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181297266187", + "endTimeUnixNano": "1795163181298111758", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "2vJ1vQSVzvY=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181297535548", + "endTimeUnixNano": "1795163181298447626", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "dGtyCbA8d1o=", + "parentSpanId": "2vJ1vQSVzvY=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181297562443", + "endTimeUnixNano": "1795163181298196778", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "R8bK9JbeBqY=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181301022534", + "endTimeUnixNano": "1795163181302309300", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "cGE4aaQh/z4=", + "parentSpanId": "R8bK9JbeBqY=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181301058435", + "endTimeUnixNano": "1795163181301969929", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "18f0Hz1jKpo=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181301291854", + "endTimeUnixNano": "1795163181302140011", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "j5BnlpBFnzs=", + "parentSpanId": "18f0Hz1jKpo=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181301313120", + "endTimeUnixNano": "1795163181301813149", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "LwmQdMzx/Kk=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181304825457", + "endTimeUnixNano": "1795163181305812381", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "EqtPPKVQfN4=", + "parentSpanId": "LwmQdMzx/Kk=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181304863068", + "endTimeUnixNano": "1795163181305624328", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "bOpJr4GcUxI=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181305063797", + "endTimeUnixNano": "1795163181306043173", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ROQ91j6NZZ8=", + "parentSpanId": "bOpJr4GcUxI=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181305085187", + "endTimeUnixNano": "1795163181305896901", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "mqmcCUTJOUw=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181308344008", + "endTimeUnixNano": "1795163181309371503", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "mTRYryPInIE=", + "parentSpanId": "mqmcCUTJOUw=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181308374572", + "endTimeUnixNano": "1795163181309206299", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ORmxOdIWIOw=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181308511671", + "endTimeUnixNano": "1795163181309481082", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "a61aM67LFI4=", + "parentSpanId": "ORmxOdIWIOw=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181308526807", + "endTimeUnixNano": "1795163181309056691", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "553mCS6Xm7E=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181311822696", + "endTimeUnixNano": "1795163181312913236", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "Y2pCKV9Qle4=", + "parentSpanId": "553mCS6Xm7E=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181311855470", + "endTimeUnixNano": "1795163181312694161", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "nK1FdeL1MVE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181312160733", + "endTimeUnixNano": "1795163181313119219", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "rxCMd+B+J1M=", + "parentSpanId": "nK1FdeL1MVE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181312188628", + "endTimeUnixNano": "1795163181312983120", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "qFE9uvWQPEU=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181315969994", + "endTimeUnixNano": "1795163181317012207", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "mMUiRsJF5yY=", + "parentSpanId": "qFE9uvWQPEU=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181316009564", + "endTimeUnixNano": "1795163181316728168", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "pBpOJlA1H0A=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181316316203", + "endTimeUnixNano": "1795163181317220942", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "FVj8LCcbOJs=", + "parentSpanId": "pBpOJlA1H0A=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181316337343", + "endTimeUnixNano": "1795163181317092182", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "t+cVr1L0dkc=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181319452726", + "endTimeUnixNano": "1795163181320550230", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "EWYfyoCBMnU=", + "parentSpanId": "t+cVr1L0dkc=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181319477703", + "endTimeUnixNano": "1795163181320363512", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "HyisMMqdpUQ=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181319845011", + "endTimeUnixNano": "1795163181320981042", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "ocKHQL/k1as=", + "parentSpanId": "HyisMMqdpUQ=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181319899050", + "endTimeUnixNano": "1795163181320741035", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "GcD1VDquVvo=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181323139690", + "endTimeUnixNano": "1795163181324335932", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "jyQ5DhaeUf8=", + "parentSpanId": "GcD1VDquVvo=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181323178969", + "endTimeUnixNano": "1795163181324184239", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "f52RA6I5rME=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181323627961", + "endTimeUnixNano": "1795163181324425414", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "JPZBomgwSS0=", + "parentSpanId": "f52RA6I5rME=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181323647600", + "endTimeUnixNano": "1795163181324102972", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "4VyjKvR+MiE=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181326554874", + "endTimeUnixNano": "1795163181327679731", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "vOdwDxuOido=", + "parentSpanId": "4VyjKvR+MiE=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181326583186", + "endTimeUnixNano": "1795163181327532833", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "P3UUZEmed+w=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "GET /api/v1/users/me", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795163181326997111", + "endTimeUnixNano": "1795163181327813495", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/users/me" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "200" + } + } + ], + "status": {} + }, + { + "traceId": "KFrpUCrqRC+Z/i3qJ+cnGQ==", + "spanId": "W4CaB5NCoaE=", + "parentSpanId": "P3UUZEmed+w=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795163181327018460", + "endTimeUnixNano": "1795163181327446229", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + } + ] + } + ] + } + ] + }, + "diagnosis": "Server 5xx spans contain slow/error PostgreSQL transaction; health remains available; sessions lock is the bottleneck." + } +} diff --git a/evidence/observability-cost.json b/evidence/observability-cost.json new file mode 100644 index 0000000..f5ce401 --- /dev/null +++ b/evidence/observability-cost.json @@ -0,0 +1,1692 @@ +{ + "method": "Fixed arrival mixed requests, 5% image uploads, 15% messages, 80% profile reads; same fixture/resources. No transparent retries.", + "notes": [ + "CPU cgroup total across load plus exporter drain; includes renderer.", + "Anonymous/working-set memory sampled at 1Hz through read-only Docker stats API, not exact per-PID peak RSS.", + "Collector rx bytes include OTLP HTTP, Prometheus scrapes and backend replies; accepted signal counts are exact Collector counter deltas.", + "Fresh API/worker/dispatcher containers per setting; data/backends retained. Per-instance counter deltas ignore expired old series.", + "Initial post-window SQL-lock probes are classified by their actual HTTP status; Docker exec startup consumed part of the hold time. The separate sampling-retention.json study synchronizes on pg_locks and guarantees 3x503 + 1xslow200 per repeat." + ], + "runs": [ + { + "mode": "off", + "enabled": 0, + "sample_ratio": 0, + "repeat": 0, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.975265541, + "successful_rps": 30.037147629826347, + "p50_seconds": 0.00300129199999688, + "p95_seconds": 0.008791667000000558, + "p99_seconds": 0.014328458000001376, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.397554, + "peak_sampled_anonymous_bytes": 178855936, + "peak_sampled_working_set_bytes": 187908096, + "rx_bytes": 4530061 + }, + "broker": { + "cpu_seconds": 0.15378800000000004, + "peak_sampled_anonymous_bytes": 4116480, + "peak_sampled_working_set_bytes": 9412608, + "rx_bytes": 43590 + }, + "dispatcher": { + "cpu_seconds": 0.6928389999999999, + "peak_sampled_anonymous_bytes": 44990464, + "peak_sampled_working_set_bytes": 47325184, + "rx_bytes": 458002 + }, + "grafana": { + "cpu_seconds": 0.04605400000000026, + "peak_sampled_anonymous_bytes": 83767296, + "peak_sampled_working_set_bytes": 89759744, + "rx_bytes": 84 + }, + "lb": { + "cpu_seconds": 0.24711599999999967, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18386944, + "rx_bytes": 4325424 + }, + "loki": { + "cpu_seconds": 0.1151819999999999, + "peak_sampled_anonymous_bytes": 62185472, + "peak_sampled_working_set_bytes": 63283200, + "rx_bytes": 370 + }, + "otel-collector": { + "cpu_seconds": 0.04942599999999997, + "peak_sampled_anonymous_bytes": 58355712, + "peak_sampled_working_set_bytes": 63848448, + "rx_bytes": 14437 + }, + "postgres": { + "cpu_seconds": 1.5883010000000013, + "peak_sampled_anonymous_bytes": 15314944, + "peak_sampled_working_set_bytes": 88096768, + "rx_bytes": 1178507 + }, + "prometheus": { + "cpu_seconds": 0.025018000000000096, + "peak_sampled_anonymous_bytes": 30494720, + "peak_sampled_working_set_bytes": 43393024, + "rx_bytes": 64749 + }, + "redis": { + "cpu_seconds": 0.16833299999999962, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14114816, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.44473300000000027, + "peak_sampled_anonymous_bytes": 69632000, + "peak_sampled_working_set_bytes": 75223040, + "rx_bytes": 11145201 + }, + "s3-store": { + "cpu_seconds": 0.4075490000000004, + "peak_sampled_anonymous_bytes": 137076736, + "peak_sampled_working_set_bytes": 241938432, + "rx_bytes": 7487615 + }, + "tempo": { + "cpu_seconds": 0.061722, + "peak_sampled_anonymous_bytes": 270999552, + "peak_sampled_working_set_bytes": 272510976, + "rx_bytes": 506 + }, + "worker": { + "cpu_seconds": 5.2742759999999995, + "peak_sampled_anonymous_bytes": 80060416, + "peak_sampled_working_set_bytes": 84627456, + "rx_bytes": 3739196 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 0, + "otelcol_receiver_accepted_log_records_total": 0, + "otelcol_receiver_accepted_metric_points_total": 0, + "otelcol_exporter_sent_spans_total": 0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "5c230696e6c348e490b5870477deb25f", + "status": 200, + "latency_seconds": 0.00486374999999839, + "trace_sampled": false + }, + { + "request_id": "0893cd3644994ecf91a14845d1034676", + "status": 200, + "latency_seconds": 0.003821957999996073, + "trace_sampled": false + }, + { + "request_id": "73d9ece550724c04890a28ba8990b480", + "status": 200, + "latency_seconds": 0.005794915999999262, + "trace_sampled": false + }, + { + "request_id": "db40195042b4493eb1e44d7dc20b96ba", + "status": 200, + "latency_seconds": 0.006047958999999992, + "trace_sampled": false + } + ] + }, + { + "mode": "full", + "enabled": 1, + "sample_ratio": 1, + "repeat": 0, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.971939667, + "successful_rps": 30.042149636141296, + "p50_seconds": 0.002889416000002143, + "p95_seconds": 0.010209084000010193, + "p99_seconds": 0.028965249999998832, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 3.328923, + "peak_sampled_anonymous_bytes": 155086848, + "peak_sampled_working_set_bytes": 163901440, + "rx_bytes": 4581068 + }, + "broker": { + "cpu_seconds": 0.17730200000000007, + "peak_sampled_anonymous_bytes": 4132864, + "peak_sampled_working_set_bytes": 9302016, + "rx_bytes": 45306 + }, + "dispatcher": { + "cpu_seconds": 0.8315530000000001, + "peak_sampled_anonymous_bytes": 45432832, + "peak_sampled_working_set_bytes": 47874048, + "rx_bytes": 482273 + }, + "grafana": { + "cpu_seconds": 0.06061899999999998, + "peak_sampled_anonymous_bytes": 83779584, + "peak_sampled_working_set_bytes": 89772032, + "rx_bytes": 84 + }, + "lb": { + "cpu_seconds": 0.2562519999999999, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18386944, + "rx_bytes": 4286332 + }, + "loki": { + "cpu_seconds": 0.18902600000000014, + "peak_sampled_anonymous_bytes": 65634304, + "peak_sampled_working_set_bytes": 66740224, + "rx_bytes": 108942 + }, + "otel-collector": { + "cpu_seconds": 0.19410300000000014, + "peak_sampled_anonymous_bytes": 57430016, + "peak_sampled_working_set_bytes": 62943232, + "rx_bytes": 1092634 + }, + "postgres": { + "cpu_seconds": 1.645151999999996, + "peak_sampled_anonymous_bytes": 16187392, + "peak_sampled_working_set_bytes": 89083904, + "rx_bytes": 1197385 + }, + "prometheus": { + "cpu_seconds": 0.04396699999999987, + "peak_sampled_anonymous_bytes": 30601216, + "peak_sampled_working_set_bytes": 43503616, + "rx_bytes": 67364 + }, + "redis": { + "cpu_seconds": 0.21558100000000024, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14110720, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.4803259999999998, + "peak_sampled_anonymous_bytes": 70320128, + "peak_sampled_working_set_bytes": 75911168, + "rx_bytes": 11145738 + }, + "s3-store": { + "cpu_seconds": 0.473001, + "peak_sampled_anonymous_bytes": 146186240, + "peak_sampled_working_set_bytes": 251293696, + "rx_bytes": 7488458 + }, + "tempo": { + "cpu_seconds": 0.08789200000000003, + "peak_sampled_anonymous_bytes": 285114368, + "peak_sampled_working_set_bytes": 286642176, + "rx_bytes": 89778 + }, + "worker": { + "cpu_seconds": 5.533063, + "peak_sampled_anonymous_bytes": 77615104, + "peak_sampled_working_set_bytes": 82518016, + "rx_bytes": 3768611 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 1950.0, + "otelcol_receiver_accepted_log_records_total": 840.0, + "otelcol_receiver_accepted_metric_points_total": 1011.0, + "otelcol_exporter_sent_spans_total": 1950.0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 1950.0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "723eac4a47374627beac821a9091b1a8", + "status": 200, + "latency_seconds": 0.007174290999998334, + "trace_sampled": true, + "trace_id": "cffabf0fc53c5709f7d40b18204dfd3b" + }, + { + "request_id": "57a2700ea7f649869da7efab6360ef81", + "status": 503, + "latency_seconds": 1.4987169580000028, + "trace_sampled": true, + "trace_id": "33caaa68b816a492a810f84630616a7a" + }, + { + "request_id": "719f8df8ee3143be8b73676766611b58", + "status": 200, + "latency_seconds": 0.007091125000002307, + "trace_sampled": true, + "trace_id": "7dad2fe9e5ad5a1bbd567765ab64dd46" + }, + { + "request_id": "f3b88188679d44f18893ab04e41a26b4", + "status": 200, + "latency_seconds": 0.005517166999993606, + "trace_sampled": true, + "trace_id": "4ea7ac61e1caaa90d2b31e514698a8da" + } + ] + }, + { + "mode": "sample10", + "enabled": 1, + "sample_ratio": 0.1, + "repeat": 0, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.972699833000007, + "successful_rps": 30.041006224338613, + "p50_seconds": 0.0027112500000043838, + "p95_seconds": 0.00865920799999742, + "p99_seconds": 0.013731374999991885, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.523611, + "peak_sampled_anonymous_bytes": 176848896, + "peak_sampled_working_set_bytes": 186454016, + "rx_bytes": 4571688 + }, + "broker": { + "cpu_seconds": 0.15326200000000068, + "peak_sampled_anonymous_bytes": 3239936, + "peak_sampled_working_set_bytes": 8257536, + "rx_bytes": 45750 + }, + "dispatcher": { + "cpu_seconds": 0.730053, + "peak_sampled_anonymous_bytes": 45420544, + "peak_sampled_working_set_bytes": 47857664, + "rx_bytes": 471320 + }, + "grafana": { + "cpu_seconds": 0.09900699999999985, + "peak_sampled_anonymous_bytes": 84144128, + "peak_sampled_working_set_bytes": 90238976, + "rx_bytes": 5856 + }, + "lb": { + "cpu_seconds": 0.23503000000000007, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18407424, + "rx_bytes": 4285664 + }, + "loki": { + "cpu_seconds": 0.1507130000000001, + "peak_sampled_anonymous_bytes": 72830976, + "peak_sampled_working_set_bytes": 73965568, + "rx_bytes": 110022 + }, + "otel-collector": { + "cpu_seconds": 0.16146100000000008, + "peak_sampled_anonymous_bytes": 61222912, + "peak_sampled_working_set_bytes": 66953216, + "rx_bytes": 738170 + }, + "postgres": { + "cpu_seconds": 1.5299630000000022, + "peak_sampled_anonymous_bytes": 14110720, + "peak_sampled_working_set_bytes": 86786048, + "rx_bytes": 1168273 + }, + "prometheus": { + "cpu_seconds": 0.03131799999999996, + "peak_sampled_anonymous_bytes": 31059968, + "peak_sampled_working_set_bytes": 43962368, + "rx_bytes": 93821 + }, + "redis": { + "cpu_seconds": 0.1691539999999998, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14114816, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.4351940000000001, + "peak_sampled_anonymous_bytes": 70418432, + "peak_sampled_working_set_bytes": 76009472, + "rx_bytes": 11144860 + }, + "s3-store": { + "cpu_seconds": 0.39130299999999973, + "peak_sampled_anonymous_bytes": 147214336, + "peak_sampled_working_set_bytes": 252542976, + "rx_bytes": 7486024 + }, + "tempo": { + "cpu_seconds": 0.07223400000000002, + "peak_sampled_anonymous_bytes": 167501824, + "peak_sampled_working_set_bytes": 169185280, + "rx_bytes": 23025 + }, + "worker": { + "cpu_seconds": 5.2699810000000005, + "peak_sampled_anonymous_bytes": 76886016, + "peak_sampled_working_set_bytes": 81866752, + "rx_bytes": 3758994 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 176.0, + "otelcol_receiver_accepted_log_records_total": 840.0, + "otelcol_receiver_accepted_metric_points_total": 956.0, + "otelcol_exporter_sent_spans_total": 176.0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 176.0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "ffcece7a06fb4cd6b066e95e4103df70", + "status": 200, + "latency_seconds": 0.003515332999995735, + "trace_sampled": false, + "trace_id": "c4dc83fcda1c1ed05c81c8e1fc22d83a" + }, + { + "request_id": "b36d8dd933364080a5fec3f8741e6fe5", + "status": 503, + "latency_seconds": 1.5039899999999875, + "trace_sampled": false, + "trace_id": "975ddde505a8354dc1754074bea788d4" + }, + { + "request_id": "e879d9d8247b42c1bcf8ab6c87a75083", + "status": 200, + "latency_seconds": 0.00465941599998132, + "trace_sampled": false, + "trace_id": "8faa8d99efcfc13c1e0d1cb504b842e6" + }, + { + "request_id": "89c845e71c7c4de28af6b9eaca054987", + "status": 200, + "latency_seconds": 0.6065338749999967, + "trace_sampled": false, + "trace_id": "5fda13d81e8a026e1b50f10a6064968c" + } + ] + }, + { + "mode": "sample25", + "enabled": 1, + "sample_ratio": 0.25, + "repeat": 0, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.97326854100001, + "successful_rps": 30.040150853044082, + "p50_seconds": 0.0028638750000027358, + "p95_seconds": 0.008952082999996946, + "p99_seconds": 0.014915040999994744, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.5460709999999995, + "peak_sampled_anonymous_bytes": 167587840, + "peak_sampled_working_set_bytes": 176685056, + "rx_bytes": 4573821 + }, + "broker": { + "cpu_seconds": 0.16021000000000019, + "peak_sampled_anonymous_bytes": 3235840, + "peak_sampled_working_set_bytes": 8249344, + "rx_bytes": 45750 + }, + "dispatcher": { + "cpu_seconds": 0.7868260000000001, + "peak_sampled_anonymous_bytes": 45428736, + "peak_sampled_working_set_bytes": 48144384, + "rx_bytes": 468546 + }, + "grafana": { + "cpu_seconds": 0.04848399999999975, + "peak_sampled_anonymous_bytes": 84168704, + "peak_sampled_working_set_bytes": 90267648, + "rx_bytes": 234 + }, + "lb": { + "cpu_seconds": 0.2286579999999998, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18644992, + "rx_bytes": 4284384 + }, + "loki": { + "cpu_seconds": 0.14923700000000029, + "peak_sampled_anonymous_bytes": 75874304, + "peak_sampled_working_set_bytes": 77029376, + "rx_bytes": 109617 + }, + "otel-collector": { + "cpu_seconds": 0.17502400000000007, + "peak_sampled_anonymous_bytes": 61571072, + "peak_sampled_working_set_bytes": 67301376, + "rx_bytes": 804572 + }, + "postgres": { + "cpu_seconds": 1.5768600000000035, + "peak_sampled_anonymous_bytes": 14389248, + "peak_sampled_working_set_bytes": 87543808, + "rx_bytes": 1181342 + }, + "prometheus": { + "cpu_seconds": 0.03631899999999999, + "peak_sampled_anonymous_bytes": 31588352, + "peak_sampled_working_set_bytes": 44494848, + "rx_bytes": 90971 + }, + "redis": { + "cpu_seconds": 0.1754850000000001, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14110720, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.4327329999999998, + "peak_sampled_anonymous_bytes": 70475776, + "peak_sampled_working_set_bytes": 76066816, + "rx_bytes": 11142662 + }, + "s3-store": { + "cpu_seconds": 0.3934250000000006, + "peak_sampled_anonymous_bytes": 159215616, + "peak_sampled_working_set_bytes": 264777728, + "rx_bytes": 7488042 + }, + "tempo": { + "cpu_seconds": 0.10833899999999996, + "peak_sampled_anonymous_bytes": 166371328, + "peak_sampled_working_set_bytes": 168288256, + "rx_bytes": 35696 + }, + "worker": { + "cpu_seconds": 5.127224999999999, + "peak_sampled_anonymous_bytes": 78036992, + "peak_sampled_working_set_bytes": 82710528, + "rx_bytes": 3760839 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 441.0, + "otelcol_receiver_accepted_log_records_total": 840.0, + "otelcol_receiver_accepted_metric_points_total": 956.0, + "otelcol_exporter_sent_spans_total": 441.0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 441.0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "a039b96a91724972a86115bde3234c3a", + "status": 503, + "latency_seconds": 1.493530667000016, + "trace_sampled": true, + "trace_id": "5ed8acb1f3d78ada379de7c77298115c" + }, + { + "request_id": "5dbafb887e7545179be21477bc5038f6", + "status": 200, + "latency_seconds": 0.0059597919999987425, + "trace_sampled": false, + "trace_id": "80eee5c8e270a673405e52b07e79ef4c" + }, + { + "request_id": "5f84c3bb362d4eae9fa885ecd5c26dd9", + "status": 503, + "latency_seconds": 1.4955291660000114, + "trace_sampled": false, + "trace_id": "7ad13402acb0d2ba498fc8a34458d54d" + }, + { + "request_id": "ed34bc1ab26b4dd8857ded5854ce7e49", + "status": 200, + "latency_seconds": 0.5992442920000087, + "trace_sampled": false, + "trace_id": "3ad0b8c978d2576caa4642e1dfe51a18" + } + ] + }, + { + "mode": "full", + "enabled": 1, + "sample_ratio": 1, + "repeat": 1, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.971833499999974, + "successful_rps": 30.042309335294668, + "p50_seconds": 0.002800540999999157, + "p95_seconds": 0.008718707999975095, + "p99_seconds": 0.014413875000002463, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.640132, + "peak_sampled_anonymous_bytes": 167768064, + "peak_sampled_working_set_bytes": 176771072, + "rx_bytes": 4577482 + }, + "broker": { + "cpu_seconds": 0.14702400000000004, + "peak_sampled_anonymous_bytes": 3235840, + "peak_sampled_working_set_bytes": 8257536, + "rx_bytes": 45750 + }, + "dispatcher": { + "cpu_seconds": 0.770845, + "peak_sampled_anonymous_bytes": 45436928, + "peak_sampled_working_set_bytes": 48349184, + "rx_bytes": 474731 + }, + "grafana": { + "cpu_seconds": 0.040907999999999944, + "peak_sampled_anonymous_bytes": 84176896, + "peak_sampled_working_set_bytes": 90267648, + "rx_bytes": 84 + }, + "lb": { + "cpu_seconds": 0.22938800000000015, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18636800, + "rx_bytes": 4283674 + }, + "loki": { + "cpu_seconds": 0.152698, + "peak_sampled_anonymous_bytes": 88342528, + "peak_sampled_working_set_bytes": 89530368, + "rx_bytes": 109077 + }, + "otel-collector": { + "cpu_seconds": 0.182226, + "peak_sampled_anonymous_bytes": 61603840, + "peak_sampled_working_set_bytes": 67334144, + "rx_bytes": 1084156 + }, + "postgres": { + "cpu_seconds": 1.5757549999999938, + "peak_sampled_anonymous_bytes": 14557184, + "peak_sampled_working_set_bytes": 87969792, + "rx_bytes": 1185855 + }, + "prometheus": { + "cpu_seconds": 0.03547000000000011, + "peak_sampled_anonymous_bytes": 33968128, + "peak_sampled_working_set_bytes": 46882816, + "rx_bytes": 88548 + }, + "redis": { + "cpu_seconds": 0.16059299999999954, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14110720, + "rx_bytes": 119760 + }, + "s3": { + "cpu_seconds": 0.42028, + "peak_sampled_anonymous_bytes": 70524928, + "peak_sampled_working_set_bytes": 76115968, + "rx_bytes": 11142331 + }, + "s3-store": { + "cpu_seconds": 0.385713, + "peak_sampled_anonymous_bytes": 160899072, + "peak_sampled_working_set_bytes": 266690560, + "rx_bytes": 7487610 + }, + "tempo": { + "cpu_seconds": 0.1287790000000001, + "peak_sampled_anonymous_bytes": 175345664, + "peak_sampled_working_set_bytes": 177360896, + "rx_bytes": 90092 + }, + "worker": { + "cpu_seconds": 5.233713, + "peak_sampled_anonymous_bytes": 78712832, + "peak_sampled_working_set_bytes": 83640320, + "rx_bytes": 3767596 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 1950.0, + "otelcol_receiver_accepted_log_records_total": 840.0, + "otelcol_receiver_accepted_metric_points_total": 968.0, + "otelcol_exporter_sent_spans_total": 1950.0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 1950.0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "33a10f26c5144d20a848d29c7150b8e4", + "status": 200, + "latency_seconds": 0.005709916999990128, + "trace_sampled": true, + "trace_id": "80caa7fccdcaedd3cbe374cb0d38539e" + }, + { + "request_id": "da630ecd36cb4471b9b36d9d3efee4c9", + "status": 503, + "latency_seconds": 1.6010425839999982, + "trace_sampled": true, + "trace_id": "d46dfa832b9aef901d583d1835729ea5" + }, + { + "request_id": "a40b3ae805114393b29084b2ba9d4efd", + "status": 200, + "latency_seconds": 0.006991167000023779, + "trace_sampled": true, + "trace_id": "1af657ca6ff44d6d5bfc858fe2952c73" + }, + { + "request_id": "4afb5c4b242944d5bdb1ca1c31878594", + "status": 200, + "latency_seconds": 0.006187374999996109, + "trace_sampled": true, + "trace_id": "e1b1f1d0d446379cc7807fc0b45b89e0" + } + ] + }, + { + "mode": "sample10", + "enabled": 1, + "sample_ratio": 0.1, + "repeat": 1, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.969775749999997, + "successful_rps": 30.04540499159086, + "p50_seconds": 0.002478499999995165, + "p95_seconds": 0.009247708000032162, + "p99_seconds": 0.012284707999981492, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.549156, + "peak_sampled_anonymous_bytes": 184029184, + "peak_sampled_working_set_bytes": 193073152, + "rx_bytes": 4573747 + }, + "broker": { + "cpu_seconds": 0.1458149999999998, + "peak_sampled_anonymous_bytes": 3235840, + "peak_sampled_working_set_bytes": 8257536, + "rx_bytes": 46194 + }, + "dispatcher": { + "cpu_seconds": 0.6954319999999999, + "peak_sampled_anonymous_bytes": 45428736, + "peak_sampled_working_set_bytes": 47878144, + "rx_bytes": 475787 + }, + "grafana": { + "cpu_seconds": 0.04330299999999987, + "peak_sampled_anonymous_bytes": 84189184, + "peak_sampled_working_set_bytes": 90279936, + "rx_bytes": 84 + }, + "lb": { + "cpu_seconds": 0.22905600000000081, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18382848, + "rx_bytes": 4285770 + }, + "loki": { + "cpu_seconds": 0.15075600000000033, + "peak_sampled_anonymous_bytes": 90304512, + "peak_sampled_working_set_bytes": 91512832, + "rx_bytes": 110502 + }, + "otel-collector": { + "cpu_seconds": 0.17933100000000035, + "peak_sampled_anonymous_bytes": 61636608, + "peak_sampled_working_set_bytes": 67366912, + "rx_bytes": 766338 + }, + "postgres": { + "cpu_seconds": 1.4643060000000006, + "peak_sampled_anonymous_bytes": 15458304, + "peak_sampled_working_set_bytes": 89055232, + "rx_bytes": 1196312 + }, + "prometheus": { + "cpu_seconds": 0.03083099999999983, + "peak_sampled_anonymous_bytes": 34013184, + "peak_sampled_working_set_bytes": 46931968, + "rx_bytes": 88528 + }, + "redis": { + "cpu_seconds": 0.16615800000000025, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14114816, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.45296800000000026, + "peak_sampled_anonymous_bytes": 70594560, + "peak_sampled_working_set_bytes": 76312576, + "rx_bytes": 11144351 + }, + "s3-store": { + "cpu_seconds": 0.41154899999999905, + "peak_sampled_anonymous_bytes": 169603072, + "peak_sampled_working_set_bytes": 275623936, + "rx_bytes": 7489492 + }, + "tempo": { + "cpu_seconds": 0.18866799999999984, + "peak_sampled_anonymous_bytes": 263524352, + "peak_sampled_working_set_bytes": 265744384, + "rx_bytes": 27438 + }, + "worker": { + "cpu_seconds": 5.451653, + "peak_sampled_anonymous_bytes": 78204928, + "peak_sampled_working_set_bytes": 82956288, + "rx_bytes": 3761278 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 253.0, + "otelcol_receiver_accepted_log_records_total": 840.0, + "otelcol_receiver_accepted_metric_points_total": 987.0, + "otelcol_exporter_sent_spans_total": 253.0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 253.0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "f855f14c71f446b4b1b05738ba4a3548", + "status": 200, + "latency_seconds": 0.00400887499995406, + "trace_sampled": false, + "trace_id": "45fd1ec22cb5838e2ac74ab2a8bcffd2" + }, + { + "request_id": "421dc6f8e82d4df79851e9edf53a9db2", + "status": 200, + "latency_seconds": 0.0038054579999879934, + "trace_sampled": false, + "trace_id": "58884c774148e2528053163d80eaca36" + }, + { + "request_id": "fabe720f47f44d6d8cdd6aff17fa52da", + "status": 200, + "latency_seconds": 0.00652137499997707, + "trace_sampled": false, + "trace_id": "fee34bd08205389767462e2c7b7549a8" + }, + { + "request_id": "8240a329f3f741ff9ea0f577f2820c39", + "status": 200, + "latency_seconds": 0.005179457999986425, + "trace_sampled": false, + "trace_id": "a8a3932c6d2b501a566bb82811d56f85" + } + ] + }, + { + "mode": "sample25", + "enabled": 1, + "sample_ratio": 0.25, + "repeat": 1, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.97339687499999, + "successful_rps": 30.039957837667522, + "p50_seconds": 0.003318040999999994, + "p95_seconds": 0.011264500000038424, + "p99_seconds": 0.015699083000015435, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.7679620000000003, + "peak_sampled_anonymous_bytes": 154787840, + "peak_sampled_working_set_bytes": 163663872, + "rx_bytes": 4576911 + }, + "broker": { + "cpu_seconds": 0.17276000000000025, + "peak_sampled_anonymous_bytes": 3235840, + "peak_sampled_working_set_bytes": 8515584, + "rx_bytes": 45750 + }, + "dispatcher": { + "cpu_seconds": 0.830878, + "peak_sampled_anonymous_bytes": 45432832, + "peak_sampled_working_set_bytes": 47878144, + "rx_bytes": 473183 + }, + "grafana": { + "cpu_seconds": 0.0578350000000003, + "peak_sampled_anonymous_bytes": 84205568, + "peak_sampled_working_set_bytes": 90296320, + "rx_bytes": 84 + }, + "lb": { + "cpu_seconds": 0.27045900000000067, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18382848, + "rx_bytes": 4285746 + }, + "loki": { + "cpu_seconds": 0.19686899999999952, + "peak_sampled_anonymous_bytes": 94715904, + "peak_sampled_working_set_bytes": 95989760, + "rx_bytes": 109961 + }, + "otel-collector": { + "cpu_seconds": 0.2040059999999997, + "peak_sampled_anonymous_bytes": 61665280, + "peak_sampled_working_set_bytes": 67395584, + "rx_bytes": 822103 + }, + "postgres": { + "cpu_seconds": 1.7547759999999997, + "peak_sampled_anonymous_bytes": 15122432, + "peak_sampled_working_set_bytes": 88715264, + "rx_bytes": 1163538 + }, + "prometheus": { + "cpu_seconds": 0.03824000000000005, + "peak_sampled_anonymous_bytes": 35225600, + "peak_sampled_working_set_bytes": 48283648, + "rx_bytes": 87631 + }, + "redis": { + "cpu_seconds": 0.1955410000000004, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14110720, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.4847110000000008, + "peak_sampled_anonymous_bytes": 70668288, + "peak_sampled_working_set_bytes": 76337152, + "rx_bytes": 11145195 + }, + "s3-store": { + "cpu_seconds": 0.45569099999999985, + "peak_sampled_anonymous_bytes": 171560960, + "peak_sampled_working_set_bytes": 277803008, + "rx_bytes": 7486022 + }, + "tempo": { + "cpu_seconds": 0.07955000000000001, + "peak_sampled_anonymous_bytes": 337682432, + "peak_sampled_working_set_bytes": 339968000, + "rx_bytes": 35437 + }, + "worker": { + "cpu_seconds": 5.590355, + "peak_sampled_anonymous_bytes": 79478784, + "peak_sampled_working_set_bytes": 84197376, + "rx_bytes": 3762205 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 467.0, + "otelcol_receiver_accepted_log_records_total": 840.0, + "otelcol_receiver_accepted_metric_points_total": 1011.0, + "otelcol_exporter_sent_spans_total": 467.0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 467.0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "3e150ed6b6524aa7bd9038d8273671fd", + "status": 200, + "latency_seconds": 0.003437415999997029, + "trace_sampled": false, + "trace_id": "8a355908f3f46b435621829b8f061616" + }, + { + "request_id": "310faf864ab841da8a15755c595d0009", + "status": 200, + "latency_seconds": 0.006516916999999012, + "trace_sampled": false, + "trace_id": "8f1be5ae6bb8a76ac59f1f2b37c5a821" + }, + { + "request_id": "cc7ee3924c61494e91ae741593076c31", + "status": 200, + "latency_seconds": 0.006249749999994947, + "trace_sampled": true, + "trace_id": "56253c1d0b0dc58822128ecd06e6b929" + }, + { + "request_id": "f5dec9866b5e41a39d1a908f6315a77b", + "status": 200, + "latency_seconds": 0.007077667000032761, + "trace_sampled": false, + "trace_id": "ae7e598b3c16f4117e4fe4d97560f1ff" + } + ] + }, + { + "mode": "off", + "enabled": 0, + "sample_ratio": 0, + "repeat": 1, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.974153833999992, + "successful_rps": 30.038819415653062, + "p50_seconds": 0.0031486670000049344, + "p95_seconds": 0.009980333000044084, + "p99_seconds": 0.015300541999977213, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.3422620000000003, + "peak_sampled_anonymous_bytes": 167350272, + "peak_sampled_working_set_bytes": 176160768, + "rx_bytes": 4532506 + }, + "broker": { + "cpu_seconds": 0.1653519999999995, + "peak_sampled_anonymous_bytes": 3239936, + "peak_sampled_working_set_bytes": 8261632, + "rx_bytes": 43146 + }, + "dispatcher": { + "cpu_seconds": 0.7756369999999999, + "peak_sampled_anonymous_bytes": 44990464, + "peak_sampled_working_set_bytes": 47382528, + "rx_bytes": 454292 + }, + "grafana": { + "cpu_seconds": 0.08641399999999999, + "peak_sampled_anonymous_bytes": 84217856, + "peak_sampled_working_set_bytes": 90308608, + "rx_bytes": 84 + }, + "lb": { + "cpu_seconds": 0.25412699999999955, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18903040, + "rx_bytes": 4325890 + }, + "loki": { + "cpu_seconds": 0.12718199999999946, + "peak_sampled_anonymous_bytes": 94724096, + "peak_sampled_working_set_bytes": 96002048, + "rx_bytes": 216 + }, + "otel-collector": { + "cpu_seconds": 0.053601000000000454, + "peak_sampled_anonymous_bytes": 61677568, + "peak_sampled_working_set_bytes": 67411968, + "rx_bytes": 13027 + }, + "postgres": { + "cpu_seconds": 1.7014229999999984, + "peak_sampled_anonymous_bytes": 14426112, + "peak_sampled_working_set_bytes": 88383488, + "rx_bytes": 1174789 + }, + "prometheus": { + "cpu_seconds": 0.031208000000000125, + "peak_sampled_anonymous_bytes": 35622912, + "peak_sampled_working_set_bytes": 48680960, + "rx_bytes": 56562 + }, + "redis": { + "cpu_seconds": 0.18207400000000007, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14110720, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.4662059999999997, + "peak_sampled_anonymous_bytes": 70725632, + "peak_sampled_working_set_bytes": 76394496, + "rx_bytes": 11145979 + }, + "s3-store": { + "cpu_seconds": 0.4462469999999996, + "peak_sampled_anonymous_bytes": 171909120, + "peak_sampled_working_set_bytes": 278372352, + "rx_bytes": 7487825 + }, + "tempo": { + "cpu_seconds": 0.08142799999999983, + "peak_sampled_anonymous_bytes": 337686528, + "peak_sampled_working_set_bytes": 339976192, + "rx_bytes": 216 + }, + "worker": { + "cpu_seconds": 5.463901, + "peak_sampled_anonymous_bytes": 77950976, + "peak_sampled_working_set_bytes": 82563072, + "rx_bytes": 3738917 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 0, + "otelcol_receiver_accepted_log_records_total": 0, + "otelcol_receiver_accepted_metric_points_total": 0, + "otelcol_exporter_sent_spans_total": 0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "27e16e0cdaf24b43975a44ea4f079322", + "status": 200, + "latency_seconds": 0.0035381669999878795, + "trace_sampled": false + }, + { + "request_id": "17921941164347fa920222eff5f2c418", + "status": 200, + "latency_seconds": 0.005465167000011206, + "trace_sampled": false + }, + { + "request_id": "fb6e9dbce9b1447dad2e87af05165740", + "status": 200, + "latency_seconds": 0.007936124999957883, + "trace_sampled": false + }, + { + "request_id": "1d92099d682c40068040a9d159f6b3c9", + "status": 200, + "latency_seconds": 0.004734958000028655, + "trace_sampled": false + } + ] + }, + { + "mode": "sample10", + "enabled": 1, + "sample_ratio": 0.1, + "repeat": 2, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.972574916999974, + "successful_rps": 30.04119411209721, + "p50_seconds": 0.0030326660000241645, + "p95_seconds": 0.009149458999957005, + "p99_seconds": 0.01591166600002225, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.540737, + "peak_sampled_anonymous_bytes": 167518208, + "peak_sampled_working_set_bytes": 176734208, + "rx_bytes": 4569983 + }, + "broker": { + "cpu_seconds": 0.15188999999999986, + "peak_sampled_anonymous_bytes": 3239936, + "peak_sampled_working_set_bytes": 8261632, + "rx_bytes": 45750 + }, + "dispatcher": { + "cpu_seconds": 0.803947, + "peak_sampled_anonymous_bytes": 45387776, + "peak_sampled_working_set_bytes": 47824896, + "rx_bytes": 467293 + }, + "grafana": { + "cpu_seconds": 0.050420999999999605, + "peak_sampled_anonymous_bytes": 84238336, + "peak_sampled_working_set_bytes": 90329088, + "rx_bytes": 84 + }, + "lb": { + "cpu_seconds": 0.22421000000000024, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18382848, + "rx_bytes": 4285710 + }, + "loki": { + "cpu_seconds": 0.17452000000000023, + "peak_sampled_anonymous_bytes": 95764480, + "peak_sampled_working_set_bytes": 97017856, + "rx_bytes": 109112 + }, + "otel-collector": { + "cpu_seconds": 0.16662400000000055, + "peak_sampled_anonymous_bytes": 61689856, + "peak_sampled_working_set_bytes": 67436544, + "rx_bytes": 718356 + }, + "postgres": { + "cpu_seconds": 1.5899460000000047, + "peak_sampled_anonymous_bytes": 15716352, + "peak_sampled_working_set_bytes": 90574848, + "rx_bytes": 1185143 + }, + "prometheus": { + "cpu_seconds": 0.03482199999999991, + "peak_sampled_anonymous_bytes": 35635200, + "peak_sampled_working_set_bytes": 48697344, + "rx_bytes": 66971 + }, + "redis": { + "cpu_seconds": 0.1688029999999996, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14110720, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.4196410000000004, + "peak_sampled_anonymous_bytes": 70754304, + "peak_sampled_working_set_bytes": 76423168, + "rx_bytes": 11144409 + }, + "s3-store": { + "cpu_seconds": 0.4007069999999988, + "peak_sampled_anonymous_bytes": 172060672, + "peak_sampled_working_set_bytes": 278761472, + "rx_bytes": 7488666 + }, + "tempo": { + "cpu_seconds": 0.06864700000000012, + "peak_sampled_anonymous_bytes": 326729728, + "peak_sampled_working_set_bytes": 329039872, + "rx_bytes": 20737 + }, + "worker": { + "cpu_seconds": 5.257816, + "peak_sampled_anonymous_bytes": 78102528, + "peak_sampled_working_set_bytes": 82845696, + "rx_bytes": 3759275 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 105.0, + "otelcol_receiver_accepted_log_records_total": 840.0, + "otelcol_receiver_accepted_metric_points_total": 987.0, + "otelcol_exporter_sent_spans_total": 105.0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 105.0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "4295191b0f894e9abdf28777e3f0d575", + "status": 200, + "latency_seconds": 0.004693334000023697, + "trace_sampled": false, + "trace_id": "f1a103b24252c5fff5da5c0702517685" + }, + { + "request_id": "b3fbc6b2a5f24410b55442f6b5cbf9ce", + "status": 200, + "latency_seconds": 0.0035256669999625956, + "trace_sampled": false, + "trace_id": "e761a8c7d4a55df0bf5997229c4732b3" + }, + { + "request_id": "70ddcacb1f464c4187afbebbfb6d7191", + "status": 200, + "latency_seconds": 0.006238959000029354, + "trace_sampled": false, + "trace_id": "aa42b036d88b126fc9ceae5d8cffd9b9" + }, + { + "request_id": "93ebeafb168c4fe19503616652b80255", + "status": 200, + "latency_seconds": 0.004492083999991792, + "trace_sampled": false, + "trace_id": "0514b97e37ae3528eef9fb2d6b7ee668" + } + ] + }, + { + "mode": "sample25", + "enabled": 1, + "sample_ratio": 0.25, + "repeat": 2, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.971592207999947, + "successful_rps": 30.0426722992902, + "p50_seconds": 0.0030581659999597832, + "p95_seconds": 0.010556667000003017, + "p99_seconds": 0.017249125000034837, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.6178429999999997, + "peak_sampled_anonymous_bytes": 154996736, + "peak_sampled_working_set_bytes": 163852288, + "rx_bytes": 4576897 + }, + "broker": { + "cpu_seconds": 0.16159400000000002, + "peak_sampled_anonymous_bytes": 3235840, + "peak_sampled_working_set_bytes": 8261632, + "rx_bytes": 46194 + }, + "dispatcher": { + "cpu_seconds": 0.798959, + "peak_sampled_anonymous_bytes": 45436928, + "peak_sampled_working_set_bytes": 47874048, + "rx_bytes": 473111 + }, + "grafana": { + "cpu_seconds": 0.05419600000000013, + "peak_sampled_anonymous_bytes": 84267008, + "peak_sampled_working_set_bytes": 90603520, + "rx_bytes": 154 + }, + "lb": { + "cpu_seconds": 0.24280599999999986, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18382848, + "rx_bytes": 4283976 + }, + "loki": { + "cpu_seconds": 0.17146299999999925, + "peak_sampled_anonymous_bytes": 96968704, + "peak_sampled_working_set_bytes": 98242560, + "rx_bytes": 110007 + }, + "otel-collector": { + "cpu_seconds": 0.18281499999999973, + "peak_sampled_anonymous_bytes": 58724352, + "peak_sampled_working_set_bytes": 64479232, + "rx_bytes": 829611 + }, + "postgres": { + "cpu_seconds": 1.730197000000004, + "peak_sampled_anonymous_bytes": 14524416, + "peak_sampled_working_set_bytes": 89919488, + "rx_bytes": 1188701 + }, + "prometheus": { + "cpu_seconds": 0.04491600000000018, + "peak_sampled_anonymous_bytes": 35741696, + "peak_sampled_working_set_bytes": 48803840, + "rx_bytes": 90008 + }, + "redis": { + "cpu_seconds": 0.183643, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14110720, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.43968200000000124, + "peak_sampled_anonymous_bytes": 70795264, + "peak_sampled_working_set_bytes": 76468224, + "rx_bytes": 11144093 + }, + "s3-store": { + "cpu_seconds": 0.4230999999999998, + "peak_sampled_anonymous_bytes": 180441088, + "peak_sampled_working_set_bytes": 287334400, + "rx_bytes": 7486228 + }, + "tempo": { + "cpu_seconds": 0.07636799999999999, + "peak_sampled_anonymous_bytes": 309518336, + "peak_sampled_working_set_bytes": 311894016, + "rx_bytes": 38180 + }, + "worker": { + "cpu_seconds": 5.2202720000000005, + "peak_sampled_anonymous_bytes": 77426688, + "peak_sampled_working_set_bytes": 82075648, + "rx_bytes": 3761824 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 514.0, + "otelcol_receiver_accepted_log_records_total": 840.0, + "otelcol_receiver_accepted_metric_points_total": 1011.0, + "otelcol_exporter_sent_spans_total": 514.0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 514.0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "5aed70c2cc5b453eac6cf7e0655d7a98", + "status": 200, + "latency_seconds": 0.003553083999975115, + "trace_sampled": false, + "trace_id": "4d16fcd86d565a8147d7240bbd7fca85" + }, + { + "request_id": "789efeeb93514993a772ad0ec225c177", + "status": 200, + "latency_seconds": 0.004778250000015305, + "trace_sampled": true, + "trace_id": "90fd2e18de6896542370fb061363f27c" + }, + { + "request_id": "afb7b3b965b341f592ea0444430759d6", + "status": 200, + "latency_seconds": 0.006446417000006477, + "trace_sampled": false, + "trace_id": "497716bd475a42c3e2efad6607ad72ba" + }, + { + "request_id": "e59306bbbe304566a5d3ae2769afbf99", + "status": 200, + "latency_seconds": 0.003542833000039991, + "trace_sampled": true, + "trace_id": "454797400df8d92c1101f75f42367560" + } + ] + }, + { + "mode": "off", + "enabled": 0, + "sample_ratio": 0, + "repeat": 2, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.975750833000006, + "successful_rps": 30.03641790569384, + "p50_seconds": 0.0030265419999864207, + "p95_seconds": 0.00981791700002077, + "p99_seconds": 0.01680349999998043, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.3706709999999998, + "peak_sampled_anonymous_bytes": 154271744, + "peak_sampled_working_set_bytes": 162893824, + "rx_bytes": 4527464 + }, + "broker": { + "cpu_seconds": 0.15782899999999955, + "peak_sampled_anonymous_bytes": 3239936, + "peak_sampled_working_set_bytes": 8265728, + "rx_bytes": 43590 + }, + "dispatcher": { + "cpu_seconds": 0.761449, + "peak_sampled_anonymous_bytes": 44986368, + "peak_sampled_working_set_bytes": 47316992, + "rx_bytes": 458050 + }, + "grafana": { + "cpu_seconds": 0.06901800000000025, + "peak_sampled_anonymous_bytes": 84291584, + "peak_sampled_working_set_bytes": 90624000, + "rx_bytes": 84 + }, + "lb": { + "cpu_seconds": 0.24708500000000022, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18382848, + "rx_bytes": 4325322 + }, + "loki": { + "cpu_seconds": 0.11884099999999975, + "peak_sampled_anonymous_bytes": 99409920, + "peak_sampled_working_set_bytes": 100728832, + "rx_bytes": 216 + }, + "otel-collector": { + "cpu_seconds": 0.07021799999999967, + "peak_sampled_anonymous_bytes": 58736640, + "peak_sampled_working_set_bytes": 64471040, + "rx_bytes": 13525 + }, + "postgres": { + "cpu_seconds": 1.7200399999999973, + "peak_sampled_anonymous_bytes": 16109568, + "peak_sampled_working_set_bytes": 91168768, + "rx_bytes": 1179316 + }, + "prometheus": { + "cpu_seconds": 0.03256900000000007, + "peak_sampled_anonymous_bytes": 35774464, + "peak_sampled_working_set_bytes": 48836608, + "rx_bytes": 58692 + }, + "redis": { + "cpu_seconds": 0.1770290000000001, + "peak_sampled_anonymous_bytes": 3096576, + "peak_sampled_working_set_bytes": 14110720, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.4793459999999996, + "peak_sampled_anonymous_bytes": 70852608, + "peak_sampled_working_set_bytes": 76521472, + "rx_bytes": 11143786 + }, + "s3-store": { + "cpu_seconds": 0.45127899999999954, + "peak_sampled_anonymous_bytes": 188768256, + "peak_sampled_working_set_bytes": 295899136, + "rx_bytes": 7488189 + }, + "tempo": { + "cpu_seconds": 0.09413999999999989, + "peak_sampled_anonymous_bytes": 309526528, + "peak_sampled_working_set_bytes": 311910400, + "rx_bytes": 216 + }, + "worker": { + "cpu_seconds": 5.5170889999999995, + "peak_sampled_anonymous_bytes": 79003648, + "peak_sampled_working_set_bytes": 83582976, + "rx_bytes": 3739025 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 0, + "otelcol_receiver_accepted_log_records_total": 0, + "otelcol_receiver_accepted_metric_points_total": 0, + "otelcol_exporter_sent_spans_total": 0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "1ab6a284391a435fbe4be5202efafc76", + "status": 200, + "latency_seconds": 0.003744709000045532, + "trace_sampled": false + }, + { + "request_id": "75df49c26b53430ea8705f286b255300", + "status": 503, + "latency_seconds": 1.5000359590000016, + "trace_sampled": false + }, + { + "request_id": "fed96c6057a34feea01a663da8e4b758", + "status": 200, + "latency_seconds": 0.004121458000099665, + "trace_sampled": false + }, + { + "request_id": "86c63d0f7d354e408467c022854e40c7", + "status": 200, + "latency_seconds": 0.006441458000040257, + "trace_sampled": false + } + ] + }, + { + "mode": "full", + "enabled": 1, + "sample_ratio": 1, + "repeat": 2, + "arrival_rps": 30, + "scheduled_requests": 600, + "confirmed_requests": 600, + "load_wall_seconds": 19.972731667000062, + "successful_rps": 30.040958342786418, + "p50_seconds": 0.0028679590000137978, + "p95_seconds": 0.009695250000049782, + "p99_seconds": 0.014552875000049426, + "errors": [], + "image_jobs_ready": 30, + "resource": { + "api": { + "cpu_seconds": 2.7994320000000004, + "peak_sampled_anonymous_bytes": 154492928, + "peak_sampled_working_set_bytes": 163278848, + "rx_bytes": 4575815 + }, + "broker": { + "cpu_seconds": 0.15614999999999934, + "peak_sampled_anonymous_bytes": 3239936, + "peak_sampled_working_set_bytes": 8265728, + "rx_bytes": 45750 + }, + "dispatcher": { + "cpu_seconds": 0.7933490000000001, + "peak_sampled_anonymous_bytes": 45432832, + "peak_sampled_working_set_bytes": 48099328, + "rx_bytes": 478640 + }, + "grafana": { + "cpu_seconds": 0.051911999999999736, + "peak_sampled_anonymous_bytes": 84291584, + "peak_sampled_working_set_bytes": 90624000, + "rx_bytes": 84 + }, + "lb": { + "cpu_seconds": 0.24583200000000005, + "peak_sampled_anonymous_bytes": 7077888, + "peak_sampled_working_set_bytes": 18382848, + "rx_bytes": 4286798 + }, + "loki": { + "cpu_seconds": 0.17122700000000002, + "peak_sampled_anonymous_bytes": 99434496, + "peak_sampled_working_set_bytes": 100769792, + "rx_bytes": 108684 + }, + "otel-collector": { + "cpu_seconds": 0.21939300000000017, + "peak_sampled_anonymous_bytes": 58748928, + "peak_sampled_working_set_bytes": 64872448, + "rx_bytes": 1093250 + }, + "postgres": { + "cpu_seconds": 1.6113169999999997, + "peak_sampled_anonymous_bytes": 16871424, + "peak_sampled_working_set_bytes": 92184576, + "rx_bytes": 1162157 + }, + "prometheus": { + "cpu_seconds": 0.035209999999999964, + "peak_sampled_anonymous_bytes": 35790848, + "peak_sampled_working_set_bytes": 48857088, + "rx_bytes": 67071 + }, + "redis": { + "cpu_seconds": 0.17342299999999966, + "peak_sampled_anonymous_bytes": 4210688, + "peak_sampled_working_set_bytes": 15237120, + "rx_bytes": 119826 + }, + "s3": { + "cpu_seconds": 0.448224999999999, + "peak_sampled_anonymous_bytes": 70877184, + "peak_sampled_working_set_bytes": 76546048, + "rx_bytes": 11142216 + }, + "s3-store": { + "cpu_seconds": 0.4562039999999996, + "peak_sampled_anonymous_bytes": 189849600, + "peak_sampled_working_set_bytes": 297209856, + "rx_bytes": 7485564 + }, + "tempo": { + "cpu_seconds": 0.09863200000000028, + "peak_sampled_anonymous_bytes": 285548544, + "peak_sampled_working_set_bytes": 287961088, + "rx_bytes": 90073 + }, + "worker": { + "cpu_seconds": 5.735841, + "peak_sampled_anonymous_bytes": 77631488, + "peak_sampled_working_set_bytes": 82354176, + "rx_bytes": 3768983 + } + }, + "collector_ingest": { + "otelcol_receiver_accepted_spans_total": 1950.0, + "otelcol_receiver_accepted_log_records_total": 840.0, + "otelcol_receiver_accepted_metric_points_total": 1011.0, + "otelcol_exporter_sent_spans_total": 1950.0, + "otelcol_exporter_send_failed_spans_total": 0 + }, + "sampled_spans_finished": 1950.0, + "sampled_spans_not_accepted_after_drain": 0, + "rare_probes": [ + { + "request_id": "efc0e010f6dc4f5e8e48b180754faed4", + "status": 200, + "latency_seconds": 0.007384333000004517, + "trace_sampled": true, + "trace_id": "10b2b587431a1429e7d6388ef5bca125" + }, + { + "request_id": "be460fb9c47742b990c27d0113a5a518", + "status": 503, + "latency_seconds": 1.4986071670000456, + "trace_sampled": true, + "trace_id": "575f0fd60e3a98910c2e7ac7cee7b64f" + }, + { + "request_id": "d6bde79ae2ed4239af44220aec6ec864", + "status": 503, + "latency_seconds": 1.4987055000000282, + "trace_sampled": true, + "trace_id": "22891f0abafba21cecc670122c55fd71" + }, + { + "request_id": "7117d80c4b3c432ea08bc5f890a5585c", + "status": 200, + "latency_seconds": 0.005082291999997324, + "trace_sampled": true, + "trace_id": "6d930e564c92b0cdb743ed893a366505" + } + ] + } + ] +} diff --git a/evidence/observability-cost.png b/evidence/observability-cost.png new file mode 100644 index 0000000..72e8fa1 Binary files /dev/null and b/evidence/observability-cost.png differ diff --git a/evidence/observability-cost.svg b/evidence/observability-cost.svg new file mode 100644 index 0000000..cdc8cb4 --- /dev/null +++ b/evidence/observability-cost.svg @@ -0,0 +1,4371 @@ + + + + + + + + 2026-11-20T11:53:53.036968 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/evidence/observability-proof.txt b/evidence/observability-proof.txt new file mode 100644 index 0000000..332b187 --- /dev/null +++ b/evidence/observability-proof.txt @@ -0,0 +1,8 @@ + Container mtusi-lab6-worker-1 Stopping + Container mtusi-lab6-worker-1 Stopped + Container mtusi-lab6-s3-1 Paused + Container mtusi-lab6-worker-1 Starting + Container mtusi-lab6-worker-1 Started + Container mtusi-lab6-s3-1 Unpaused + Container mtusi-lab6-worker-1 Running +PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..4af7cfe --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18093" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/pipeline-regression.json b/evidence/pipeline-regression.json new file mode 100644 index 0000000..719c43e --- /dev/null +++ b/evidence/pipeline-regression.json @@ -0,0 +1,119 @@ +{ + "invariants": { + "commit_before_publish_recovered": true, + "publish_before_marker_duplicate_safe": true, + "crashes_before_after_side_effect_and_commit": true, + "invalid_envelope_observable_dlq": true, + "duplicate_and_reorder_no_second_effect": true, + "concurrent_duplicate_single_claim": true, + "stale_running_version_fenced": true, + "old_delivery_after_new_ready_no_overwrite": true, + "failed_registry_explicit_replay_and_ready_refusal": true, + "transient_backoff_exhaustion_then_replay": true, + "invalid_decode_permanent_failed": true, + "cleanup_preserves_current_and_fresh_failed": true, + "ready_staging_cleanup_idempotent": true + }, + "faults": [ + { + "point": "before_side_effect", + "exit_code": 77, + "state_at_crash": "processing", + "recovered": "ready", + "attempts": 2, + "result_objects": 4 + }, + { + "point": "after_original", + "exit_code": 77, + "state_at_crash": "processing", + "recovered": "ready", + "attempts": 2, + "result_objects": 4 + }, + { + "point": "after_objects", + "exit_code": 77, + "state_at_crash": "processing", + "recovered": "ready", + "attempts": 2, + "result_objects": 4 + }, + { + "point": "after_commit", + "exit_code": 77, + "state_at_crash": "ready", + "recovered": "ready", + "attempts": 1, + "result_objects": 4 + } + ], + "retry": { + "transitions": [ + { + "state": "queued", + "attempt": 1, + "error_code": "storage_unavailable" + }, + { + "state": "failed", + "attempt": 2, + "error_code": "retry_exhausted" + } + ], + "seconds": 1.983, + "replayed_attempt": 3 + }, + "transforms": [ + { + "kind": "alpha", + "size": [ + 20, + 10 + ], + "exif_removed": true + }, + { + "kind": "exif", + "size": [ + 256, + 160 + ], + "exif_removed": true + }, + { + "kind": "small", + "size": [ + 5, + 3 + ], + "exif_removed": true + } + ], + "cleanup": [ + { + "apply": false, + "retention_seconds": 86400, + "counts": { + "ready_staging": 397, + "expired_staging": 0 + } + }, + { + "apply": true, + "retention_seconds": 86400, + "counts": { + "ready_staging": 397, + "expired_staging": 0 + } + }, + { + "apply": true, + "retention_seconds": 86400, + "counts": { + "ready_staging": 0, + "expired_staging": 0 + } + } + ] +} diff --git a/evidence/restart-after.json b/evidence/restart-after.json new file mode 100644 index 0000000..15f0ecd --- /dev/null +++ b/evidence/restart-after.json @@ -0,0 +1,546 @@ +[ + { + "container": "cf2cfedf29de75b2458ec532a42f264cdb7ef71435900a7ef1c478d9d6b2e61f", + "name": "/mtusi-lab6-api-1", + "service": "api", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:45:44.122480904Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b", + "name": "/mtusi-lab6-api-2", + "service": "api", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:45:44.196103962Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "13d6ed83fc36b28aa57fc7cf302d9624098934b1f9c6ec68254a2b1a5f95228d", + "name": "/mtusi-lab6-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-11-20T08:45:30.515512908Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab6_broker_data" + } + ] + }, + { + "container": "076e71ce04ff5adbc19738e63a8ed319ad6f645183e6e65e47aec7dd97afe926", + "name": "/mtusi-lab6-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": false, + "started_at": "2026-11-20T08:45:43.045813539Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "b2de6e448b4102ff9f25833bf99cd5a9978947dae0c3e28e743c3ef18f58b943", + "name": "/mtusi-lab6-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:45:43.627780758Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "54edeaec98aff44bfe34f8c103c33694187550a9b690cde7648017ad323d7310", + "name": "/mtusi-lab6-grafana-1", + "service": "grafana", + "image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d", + "user": "472", + "running": true, + "started_at": "2026-11-20T08:45:30.734589337Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 500000000, + "ports": { + "3000/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "13006" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/etc/grafana/provisioning", + "name": null + }, + { + "type": "bind", + "destination": "/var/lib/grafana/dashboards", + "name": null + }, + { + "type": "volume", + "destination": "/var/lib/grafana", + "name": "mtusi-lab6_grafana_data" + } + ] + }, + { + "container": "7db8217af00aa7be6a7e3d19f656faf329598ef85bfc8fb2fc2d15ec0cf72c28", + "name": "/mtusi-lab6-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-11-20T08:45:49.83573792Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18093" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "712535508b37fe699452fb2a4ed62a005a8cfa7a14e2b9827a9c82cd4ec982c9", + "name": "/mtusi-lab6-loki-1", + "service": "loki", + "image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd", + "user": "10001", + "running": true, + "started_at": "2026-11-20T08:45:31.075549983Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 500000000, + "ports": { + "3100/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "13106" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/etc/loki/config.yaml", + "name": null + }, + { + "type": "volume", + "destination": "/loki", + "name": "mtusi-lab6_logs_data" + } + ] + }, + { + "container": "6374cc6d3e2b163264c18e1181ea5be5b53f365a5324686a908ea8f94861a6c7", + "name": "/mtusi-lab6-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": false, + "started_at": "2026-11-20T08:45:42.539353205Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "container": "89012420d895c4c7c0d9fdaf7c1ff40c3a5f97dca279a0ef8cec0be3903dce42", + "name": "/mtusi-lab6-otel-collector-1", + "service": "otel-collector", + "image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:45:30.652417444Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 500000000, + "ports": { + "4317/tcp": null, + "4318/tcp": null, + "55679/tcp": null, + "8888/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18896" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/etc/otel/config.yaml", + "name": null + } + ] + }, + { + "container": "2257079aebc7a27eda5607608d7c4e7df3d1a196eb2c65d7236a1b60f7466c91", + "name": "/mtusi-lab6-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-11-20T08:45:30.708099365Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab6_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "937df26581a582233f77674d7c66a0b7fbba5fffd2377977ebd13857cd510a66", + "name": "/mtusi-lab6-prometheus-1", + "service": "prometheus", + "image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996", + "user": "nobody", + "running": true, + "started_at": "2026-11-20T08:45:30.770516143Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 500000000, + "ports": { + "9090/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "19096" + } + ] + }, + "mounts": [ + { + "type": "volume", + "destination": "/prometheus", + "name": "mtusi-lab6_metrics_data" + }, + { + "type": "bind", + "destination": "/etc/prometheus/prometheus.yaml", + "name": null + }, + { + "type": "bind", + "destination": "/etc/prometheus/alerts.yaml", + "name": null + } + ] + }, + { + "container": "cc255a9bdb8967c853de2a4f167fe269a1a4f004e0b88ab174c79397212fd2a4", + "name": "/mtusi-lab6-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-11-20T08:45:30.676898647Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "db7892dbab7e8fc10b9d4431ee816406d604a540d7068b70aef208767168a178" + } + ] + }, + { + "container": "c4564618c7c266a67ed1f6a05989f8156ca42582a1860fe8d868fc758d18d4f3", + "name": "/mtusi-lab6-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-11-20T08:45:30.688051331Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "3f039af87109799a85e80069eb418dd6c13b9b036ffc46fb0e6ed1bfc21407de" + } + ] + }, + { + "container": "d3abacb0bdaf348f062e5ec77599ea8a0450872a17b450b190ba7bce78166129", + "name": "/mtusi-lab6-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-11-20T08:45:31.797812564Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab6_s3_data" + } + ] + }, + { + "container": "9cfc7a5053725737ecfc90211eed31a0c97a15aeb63ca270b9596c4af69de808", + "name": "/mtusi-lab6-tempo-1", + "service": "tempo", + "image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:45:40.551529375Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "3200/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "13206" + } + ] + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/tempo", + "name": "mtusi-lab6_traces_data" + }, + { + "type": "bind", + "destination": "/etc/tempo/config.yaml", + "name": null + } + ] + }, + { + "container": "ef14a69e994f9f18fdb3438a03a28a974a3fcfc564f2d0eedeff3949a8f9bb29", + "name": "/mtusi-lab6-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:45:44.122215389Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } +] diff --git a/evidence/restart-before.json b/evidence/restart-before.json new file mode 100644 index 0000000..348770b --- /dev/null +++ b/evidence/restart-before.json @@ -0,0 +1,546 @@ +[ + { + "container": "cf2cfedf29de75b2458ec532a42f264cdb7ef71435900a7ef1c478d9d6b2e61f", + "name": "/mtusi-lab6-api-1", + "service": "api", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:43:28.014761078Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b", + "name": "/mtusi-lab6-api-2", + "service": "api", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:43:28.105739751Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "13d6ed83fc36b28aa57fc7cf302d9624098934b1f9c6ec68254a2b1a5f95228d", + "name": "/mtusi-lab6-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-11-20T08:20:26.890287569Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab6_broker_data" + } + ] + }, + { + "container": "076e71ce04ff5adbc19738e63a8ed319ad6f645183e6e65e47aec7dd97afe926", + "name": "/mtusi-lab6-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": false, + "started_at": "2026-11-20T08:40:43.538584225Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "b2de6e448b4102ff9f25833bf99cd5a9978947dae0c3e28e743c3ef18f58b943", + "name": "/mtusi-lab6-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:43:28.015064203Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "54edeaec98aff44bfe34f8c103c33694187550a9b690cde7648017ad323d7310", + "name": "/mtusi-lab6-grafana-1", + "service": "grafana", + "image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d", + "user": "472", + "running": true, + "started_at": "2026-11-20T08:21:11.514848908Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 500000000, + "ports": { + "3000/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "13006" + } + ] + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/grafana", + "name": "mtusi-lab6_grafana_data" + }, + { + "type": "bind", + "destination": "/etc/grafana/provisioning", + "name": null + }, + { + "type": "bind", + "destination": "/var/lib/grafana/dashboards", + "name": null + } + ] + }, + { + "container": "7db8217af00aa7be6a7e3d19f656faf329598ef85bfc8fb2fc2d15ec0cf72c28", + "name": "/mtusi-lab6-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-11-20T08:20:40.887085261Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18093" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "712535508b37fe699452fb2a4ed62a005a8cfa7a14e2b9827a9c82cd4ec982c9", + "name": "/mtusi-lab6-loki-1", + "service": "loki", + "image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd", + "user": "10001", + "running": true, + "started_at": "2026-11-20T08:21:11.513864698Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 500000000, + "ports": { + "3100/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "13106" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/etc/loki/config.yaml", + "name": null + }, + { + "type": "volume", + "destination": "/loki", + "name": "mtusi-lab6_logs_data" + } + ] + }, + { + "container": "6374cc6d3e2b163264c18e1181ea5be5b53f365a5324686a908ea8f94861a6c7", + "name": "/mtusi-lab6-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": false, + "started_at": "2026-11-20T08:40:43.038416658Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "89012420d895c4c7c0d9fdaf7c1ff40c3a5f97dca279a0ef8cec0be3903dce42", + "name": "/mtusi-lab6-otel-collector-1", + "service": "otel-collector", + "image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:21:11.514654968Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 500000000, + "ports": { + "4317/tcp": null, + "4318/tcp": null, + "55679/tcp": null, + "8888/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18896" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/etc/otel/config.yaml", + "name": null + } + ] + }, + { + "container": "2257079aebc7a27eda5607608d7c4e7df3d1a196eb2c65d7236a1b60f7466c91", + "name": "/mtusi-lab6-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-11-20T08:20:26.86209481Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab6_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "937df26581a582233f77674d7c66a0b7fbba5fffd2377977ebd13857cd510a66", + "name": "/mtusi-lab6-prometheus-1", + "service": "prometheus", + "image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996", + "user": "nobody", + "running": true, + "started_at": "2026-11-20T08:21:11.515853797Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 500000000, + "ports": { + "9090/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "19096" + } + ] + }, + "mounts": [ + { + "type": "volume", + "destination": "/prometheus", + "name": "mtusi-lab6_metrics_data" + }, + { + "type": "bind", + "destination": "/etc/prometheus/prometheus.yaml", + "name": null + }, + { + "type": "bind", + "destination": "/etc/prometheus/alerts.yaml", + "name": null + } + ] + }, + { + "container": "cc255a9bdb8967c853de2a4f167fe269a1a4f004e0b88ab174c79397212fd2a4", + "name": "/mtusi-lab6-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-11-20T08:20:26.875269419Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "db7892dbab7e8fc10b9d4431ee816406d604a540d7068b70aef208767168a178" + } + ] + }, + { + "container": "c4564618c7c266a67ed1f6a05989f8156ca42582a1860fe8d868fc758d18d4f3", + "name": "/mtusi-lab6-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-11-20T08:20:31.57071865Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "3f039af87109799a85e80069eb418dd6c13b9b036ffc46fb0e6ed1bfc21407de" + } + ] + }, + { + "container": "d3abacb0bdaf348f062e5ec77599ea8a0450872a17b450b190ba7bce78166129", + "name": "/mtusi-lab6-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-11-20T08:20:26.891493875Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab6_s3_data" + } + ] + }, + { + "container": "9cfc7a5053725737ecfc90211eed31a0c97a15aeb63ca270b9596c4af69de808", + "name": "/mtusi-lab6-tempo-1", + "service": "tempo", + "image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:28:48.715669396Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "3200/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "13206" + } + ] + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/tempo", + "name": "mtusi-lab6_traces_data" + }, + { + "type": "bind", + "destination": "/etc/tempo/config.yaml", + "name": null + } + ] + }, + { + "container": "ef14a69e994f9f18fdb3438a03a28a974a3fcfc564f2d0eedeff3949a8f9bb29", + "name": "/mtusi-lab6-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab6", + "user": "10001:10001", + "running": true, + "started_at": "2026-11-20T08:45:08.489572558Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } +] diff --git a/evidence/retry-trace.json b/evidence/retry-trace.json new file mode 100644 index 0000000..0dc60b1 --- /dev/null +++ b/evidence/retry-trace.json @@ -0,0 +1,989 @@ +{ + "trace_id": "dd1cb3e17dc547bc8bd3f94033e39f7a", + "retry_observed": true, + "trace": { + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "067e0d5099614580ab61516259b76c14" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "k4UL1Qk8WSI=", + "parentSpanId": "ASNFZ4mrze8=", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795164308161361378", + "endTimeUnixNano": "1795164308169243229", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "wgvjepi5Fj0=", + "parentSpanId": "k4UL1Qk8WSI=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164308161418698", + "endTimeUnixNano": "1795164308162079443", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "6f3FdmFjcfY=", + "parentSpanId": "k4UL1Qk8WSI=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164308162156009", + "endTimeUnixNano": "1795164308162662788", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "rvhnrnpClkQ=", + "parentSpanId": "k4UL1Qk8WSI=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164308163083649", + "endTimeUnixNano": "1795164308164393283", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "qBl9xvUTfq8=", + "parentSpanId": "k4UL1Qk8WSI=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164308164428226", + "endTimeUnixNano": "1795164308168921394", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "R2wwvDNpB68=", + "parentSpanId": "qBl9xvUTfq8=", + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164308164676000", + "endTimeUnixNano": "1795164308168082844", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "put_object" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "6b019c7baab14c89976d2b4081f1c509" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-dispatcher" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "t4R0QxdjlSI=", + "parentSpanId": "qBl9xvUTfq8=", + "name": "attachment publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1795164308271913285", + "endTimeUnixNano": "1795164308274338175", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachment-jobs" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "nBeLRAx4mXA=", + "parentSpanId": "qBl9xvUTfq8=", + "name": "attachment publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1795164312922355612", + "endTimeUnixNano": "1795164312924775467", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachment-jobs" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "742be3a8d93342da9117f92711193f78" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "AqkcRaZWl50=", + "parentSpanId": "t4R0QxdjlSI=", + "name": "attachment process", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1795164309031454501", + "endTimeUnixNano": "1795164312029923986", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachment-jobs" + } + }, + { + "key": "job.id", + "value": { + "stringValue": "9d711cdf-347d-4590-91da-4908beb37202" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "cq34RtwooBc=", + "parentSpanId": "AqkcRaZWl50=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164309031484554", + "endTimeUnixNano": "1795164309038640365", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "PCx2LcCU6hc=", + "parentSpanId": "AqkcRaZWl50=", + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164309108621018", + "endTimeUnixNano": "1795164312013033788", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ReadTimeoutError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "cX5pEW5zHb0=", + "parentSpanId": "AqkcRaZWl50=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164312013553635", + "endTimeUnixNano": "1795164312029457465", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "yG0WSHlT7kQ=", + "parentSpanId": "ccEcqG6ZxPo=", + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164312924989622", + "endTimeUnixNano": "1795164312936812921", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "742be3a8d93342da9117f92711193f78" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "ccEcqG6ZxPo=", + "parentSpanId": "nBeLRAx4mXA=", + "flags": 768, + "name": "attachment process", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1795164312924913496", + "endTimeUnixNano": "1795164314747321271", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachment-jobs" + } + }, + { + "key": "job.id", + "value": { + "stringValue": "9d711cdf-347d-4590-91da-4908beb37202" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "2" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "Qf833YZm6d8=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164312943026506", + "endTimeUnixNano": "1795164314534106374", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "7QJGzOLYHEk=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.get_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314534253963", + "endTimeUnixNano": "1795164314538495180", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "get_object" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "uj8SpqBZjXM=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314714992685", + "endTimeUnixNano": "1795164314719977680", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ClientError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "RPy3F6vP7SY=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314720094322", + "endTimeUnixNano": "1795164314725722348", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "put_object" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "HvB6dqnveCs=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314725791217", + "endTimeUnixNano": "1795164314727041344", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ClientError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "DJqV5J+noKw=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314727130718", + "endTimeUnixNano": "1795164314729233570", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "put_object" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "YhhL1bvMeTM=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314729272549", + "endTimeUnixNano": "1795164314730313956", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ClientError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "92rjoPWvm5g=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314730379104", + "endTimeUnixNano": "1795164314732329971", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "put_object" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "7VymNXkyejE=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314732386665", + "endTimeUnixNano": "1795164314733725616", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ClientError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "5RGC0O8dQC8=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314733807464", + "endTimeUnixNano": "1795164314736051098", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "put_object" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "EGXBlCQ2GhE=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314736092741", + "endTimeUnixNano": "1795164314743975478", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==", + "spanId": "uADsu2p9U4s=", + "parentSpanId": "ccEcqG6ZxPo=", + "flags": 256, + "name": "s3.delete_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164314744065993", + "endTimeUnixNano": "1795164314746751040", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "delete_object" + } + } + ], + "status": {} + } + ] + } + ] + } + ] + }, + "logs": [ + { + "timestamp": "2026-11-20T08:45:12.027960+00:00", + "severity": "INFO", + "service.name": "messenger-worker", + "event": "job_retry", + "sampled": true, + "trace_id": "dd1cb3e17dc547bc8bd3f94033e39f7a", + "span_id": "717e69116e731dbd", + "job_id": "9d711cdf-347d-4590-91da-4908beb37202", + "attempt": 1, + "error_code": "storage_unavailable" + } + ] +} diff --git a/evidence/sampling-retention.json b/evidence/sampling-retention.json new file mode 100644 index 0000000..f126d72 --- /dev/null +++ b/evidence/sampling-retention.json @@ -0,0 +1,348 @@ +{ + "method": "Three repeats per full/10%/25% setting; each holds sessions lock until pg_locks confirms it. 3x503 plus one slow200 per repeat. Sampling is ordinary SDK head sampling without forced traceparent.", + "runs": [ + { + "mode": "full", + "repeat": 0, + "probes": [ + { + "request_id": "95355e81cca54f83be9a19bd527e9825", + "status": 503, + "latency_seconds": 1.5030926670000007, + "trace_sampled": true, + "trace_id": "99c07cebfe6d37a278f4e0a9c603c1c2", + "tempo_trace_found": true + }, + { + "request_id": "4e617be83d2247d0b849744fa0936deb", + "status": 503, + "latency_seconds": 1.6241221669999995, + "trace_sampled": true, + "trace_id": "806f738e3362110032113077494f8c8b", + "tempo_trace_found": true + }, + { + "request_id": "b1c982a06765483eac67aa8cf72f2be1", + "status": 503, + "latency_seconds": 1.4874708749999996, + "trace_sampled": true, + "trace_id": "a4f07c609d396e2071bb8bdbebcbad4f", + "tempo_trace_found": true + }, + { + "request_id": "822110e4f137423084355e5c8d260dcb", + "status": 200, + "latency_seconds": 1.1728969580000026, + "trace_sampled": true, + "trace_id": "8082c09a793e3535f33f203cb1dbe5da", + "tempo_trace_found": true + } + ] + }, + { + "mode": "full", + "repeat": 1, + "probes": [ + { + "request_id": "da8b813d32d144cd844124a5fa86435e", + "status": 503, + "latency_seconds": 1.4984728329999975, + "trace_sampled": true, + "trace_id": "0fe73d0ff759f1e9f2f6d9e304a8fb96", + "tempo_trace_found": true + }, + { + "request_id": "afb0eccacc314dcc8d15e6375ed1f674", + "status": 503, + "latency_seconds": 1.5027760410000006, + "trace_sampled": true, + "trace_id": "04c97d704d04c30ed712dc1a6c506639", + "tempo_trace_found": true + }, + { + "request_id": "2633bfaaa83040dc9ec3f21c968b6774", + "status": 503, + "latency_seconds": 1.6313028749999994, + "trace_sampled": true, + "trace_id": "a849e96eb0258348dba259183e9455b5", + "tempo_trace_found": true + }, + { + "request_id": "5f1a8b638d594e7e8e249d83e29a3c0a", + "status": 200, + "latency_seconds": 1.1764352089999974, + "trace_sampled": true, + "trace_id": "46f17923d4d83be8aec4863497913de9", + "tempo_trace_found": true + } + ] + }, + { + "mode": "full", + "repeat": 2, + "probes": [ + { + "request_id": "f6cf9187274f4d4494f2546fdccf9cb5", + "status": 503, + "latency_seconds": 1.4932770829999953, + "trace_sampled": true, + "trace_id": "b57747b1e2c746aafa5fb7d7e9ec7e33", + "tempo_trace_found": true + }, + { + "request_id": "5820d13764fe4ab5826a24e6be1dd12c", + "status": 503, + "latency_seconds": 1.5006451660000053, + "trace_sampled": true, + "trace_id": "f167d8d9df62e009ab0c2c3ac4ee3fc3", + "tempo_trace_found": true + }, + { + "request_id": "18e12c0c4a424141b916fd4f7c847c8b", + "status": 503, + "latency_seconds": 1.5060892499999952, + "trace_sampled": true, + "trace_id": "453c386a3fcd16e940832672ef0b072a", + "tempo_trace_found": true + }, + { + "request_id": "56df38255f974d75a68b6defb7ae7dbf", + "status": 200, + "latency_seconds": 1.181737209000005, + "trace_sampled": true, + "trace_id": "9380177eb739659ba6bfb122f4c5cb10", + "tempo_trace_found": true + } + ] + }, + { + "mode": "sample10", + "repeat": 0, + "probes": [ + { + "request_id": "7c5131770a4e4ae78594065d30041f6b", + "status": 503, + "latency_seconds": 1.5020687499999994, + "trace_sampled": false, + "trace_id": "2f49be17fa517e268808bc7655dfec57", + "tempo_trace_found": false + }, + { + "request_id": "083b4e2c54b34fb695e3e097774a02fb", + "status": 503, + "latency_seconds": 1.501452209000007, + "trace_sampled": false, + "trace_id": "6b7b3c7eb4f9d107cdef92d247489d39", + "tempo_trace_found": false + }, + { + "request_id": "d63a94e59cc74918881a6c4821129fc0", + "status": 503, + "latency_seconds": 1.5036532090000065, + "trace_sampled": false, + "trace_id": "e6b3c5a1434815bc391a764eba18ad1f", + "tempo_trace_found": false + }, + { + "request_id": "7aba69c8960a4e58ab8e1189f91e8df8", + "status": 200, + "latency_seconds": 1.1895337499999954, + "trace_sampled": true, + "trace_id": "a4b7c2e50c6db0db11b63d6988a5f581", + "tempo_trace_found": true + } + ] + }, + { + "mode": "sample10", + "repeat": 1, + "probes": [ + { + "request_id": "db9a19fe6edf44b189e4c85527305b97", + "status": 503, + "latency_seconds": 1.508617583000003, + "trace_sampled": false, + "trace_id": "43478ce0a70fc97c1f551e0b770c8450", + "tempo_trace_found": false + }, + { + "request_id": "259f18e925b54205a7f86ccf23588581", + "status": 503, + "latency_seconds": 1.6092364170000053, + "trace_sampled": false, + "trace_id": "de05c2c1da9da6e5aaad96f7562f8626", + "tempo_trace_found": false + }, + { + "request_id": "7463cb267c1b4b7c836a79772d11d304", + "status": 503, + "latency_seconds": 1.4921612919999916, + "trace_sampled": false, + "trace_id": "63f2de04875064912b355e6f7ccf2bf8", + "tempo_trace_found": false + }, + { + "request_id": "e6013482b7174a469182f9776832c5f8", + "status": 200, + "latency_seconds": 1.1786814999999962, + "trace_sampled": false, + "trace_id": "bc385f8114ac1d41fa55f762830d9a70", + "tempo_trace_found": false + } + ] + }, + { + "mode": "sample10", + "repeat": 2, + "probes": [ + { + "request_id": "9a8de42292644772a11b19bb597b98e9", + "status": 503, + "latency_seconds": 1.4827315419999962, + "trace_sampled": false, + "trace_id": "62ef4444e74027412566b41ff3330a09", + "tempo_trace_found": false + }, + { + "request_id": "444fd3c3d88c46cd89e1691852087fb3", + "status": 503, + "latency_seconds": 1.4932578330000013, + "trace_sampled": false, + "trace_id": "95d7610acd0f35487ec10e503fc12622", + "tempo_trace_found": false + }, + { + "request_id": "6f616e3942134f7797c6c0b3767c0d0e", + "status": 503, + "latency_seconds": 1.497054833000007, + "trace_sampled": false, + "trace_id": "86793b238c9e2b7e586cb02e2f8bfc42", + "tempo_trace_found": false + }, + { + "request_id": "718f724c1a8d41bfa3ec720daee432d3", + "status": 200, + "latency_seconds": 1.1874764580000061, + "trace_sampled": false, + "trace_id": "6b08c6339522e460bd77a9fd0f72438a", + "tempo_trace_found": false + } + ] + }, + { + "mode": "sample25", + "repeat": 0, + "probes": [ + { + "request_id": "5c805a3468b64739bd1c839f6ddedd56", + "status": 503, + "latency_seconds": 1.4879085000000032, + "trace_sampled": false, + "trace_id": "5cf9d06acb989768f56a26c31e6d5255", + "tempo_trace_found": false + }, + { + "request_id": "ff5596ad44c6461589e621a7b3f09676", + "status": 503, + "latency_seconds": 1.5042155419999972, + "trace_sampled": false, + "trace_id": "f8f790abc18c384d58225dceb4d1e8a9", + "tempo_trace_found": false + }, + { + "request_id": "3ee8c8db9d4a4890b866d3cb40e18bb8", + "status": 503, + "latency_seconds": 1.4965452500000112, + "trace_sampled": false, + "trace_id": "b85f2a7d3cd2ff3dd4fa66fc8b37dea6", + "tempo_trace_found": false + }, + { + "request_id": "f21c138335d64c688adb25c137c9a253", + "status": 200, + "latency_seconds": 1.1858503330000048, + "trace_sampled": true, + "trace_id": "c1df2d7ecfb8448a2f33be6756f23c38", + "tempo_trace_found": true + } + ] + }, + { + "mode": "sample25", + "repeat": 1, + "probes": [ + { + "request_id": "053ece24cfa1465d9702c67781863703", + "status": 503, + "latency_seconds": 1.5010194580000018, + "trace_sampled": true, + "trace_id": "b7c73109f24a628720011f7401fc3d42", + "tempo_trace_found": true + }, + { + "request_id": "7d30c3b85ebe480ea041d2261af90b3c", + "status": 503, + "latency_seconds": 1.4840848329999972, + "trace_sampled": false, + "trace_id": "438e9ec5155e621e5ef24df2b6e12568", + "tempo_trace_found": false + }, + { + "request_id": "8c23bd6e2acc451d8abac7c787225b06", + "status": 503, + "latency_seconds": 1.4938515830000085, + "trace_sampled": false, + "trace_id": "92267b13ab39964ae58d8bf054c2f52a", + "tempo_trace_found": false + }, + { + "request_id": "3ddb46344bae4d50989c8779264eeadf", + "status": 200, + "latency_seconds": 1.1738840829999901, + "trace_sampled": false, + "trace_id": "66d1a465ebede54e734068bc6e6092ca", + "tempo_trace_found": false + } + ] + }, + { + "mode": "sample25", + "repeat": 2, + "probes": [ + { + "request_id": "dbcb7d27ff3a4067af49f2ad559beaf7", + "status": 503, + "latency_seconds": 1.4999495829999887, + "trace_sampled": false, + "trace_id": "88fa0b73db8d67b0be62f9ef653d5c1c", + "tempo_trace_found": false + }, + { + "request_id": "1644972bdbbf4ad4ae0bb1be96d60853", + "status": 503, + "latency_seconds": 1.5033828749999998, + "trace_sampled": false, + "trace_id": "815ba97d46bc980d6df03e8542f1bd94", + "tempo_trace_found": false + }, + { + "request_id": "34ecddeb28764f6da5aae311d8297f15", + "status": 503, + "latency_seconds": 1.486551832999993, + "trace_sampled": false, + "trace_id": "5b1d99439c3aadf44db722c9543be3f3", + "tempo_trace_found": false + }, + { + "request_id": "558612f66b374e6d964b927a866bd9bf", + "status": 200, + "latency_seconds": 1.1901341250000144, + "trace_sampled": false, + "trace_id": "e6505908ffe05caf83a8a7d49f4476a0", + "tempo_trace_found": false + } + ] + } + ], + "presence_verification": "Actual Tempo trace API checked for every probe: sampled present; unsampled HTTP404." +} diff --git a/evidence/stopped-instances.jsonl b/evidence/stopped-instances.jsonl new file mode 100644 index 0000000..8e4ba37 --- /dev/null +++ b/evidence/stopped-instances.jsonl @@ -0,0 +1 @@ +{"instance": "208b43d0e3fe4d8daeccd4d252d691fe", "container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b", "running": false, "exit_code": 0, "seconds": 0.691} diff --git a/evidence/storage-security.json b/evidence/storage-security.json new file mode 100644 index 0000000..c1b3f72 --- /dev/null +++ b/evidence/storage-security.json @@ -0,0 +1,16 @@ +{ + "anonymous_object_and_list_403": true, + "job_fields": [ + "attachment_id", + "attempt", + "created_at", + "job_id", + "pipeline_version", + "schema_version", + "traceparent", + "type" + ], + "job_bytes": 317, + "no_source_bytes_or_credentials_in_envelope": true, + "strict_invalid_envelope_checks": true +} diff --git a/evidence/telemetry-persistence.json b/evidence/telemetry-persistence.json new file mode 100644 index 0000000..7e56e0d --- /dev/null +++ b/evidence/telemetry-persistence.json @@ -0,0 +1,27 @@ +{ + "trace_id": "a630912396b64101b293cc306ee44ed8", + "historical_span_count": 21, + "historical_logs": 8, + "fixed_metric_query_time": 1795164311.5720682, + "metric_before": [ + { + "metric": {}, + "value": [ + 1795164311.572, + "485" + ] + } + ], + "metric_after": [ + { + "metric": {}, + "value": [ + 1795164311.572, + "485" + ] + } + ], + "business_data_and_existing_session_survived": true, + "all_telemetry_volumes_restarted": true, + "trace_log_metric_history_survived": true +} diff --git a/evidence/trace-proof.json b/evidence/trace-proof.json new file mode 100644 index 0000000..8f789cd --- /dev/null +++ b/evidence/trace-proof.json @@ -0,0 +1,886 @@ +{ + "trace_id": "a630912396b64101b293cc306ee44ed8", + "request_id": "cb06618bb089444e8958f45beb2a8d1f", + "status": "ready", + "consumer_parent_matches_producer": true, + "producer_descends_from_upload": true, + "api_worker_logs_correlated": true, + "secret_canaries_absent": true, + "metric_labels_allowlisted": true, + "trace": { + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-dispatcher" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "6b019c7baab14c89976d2b4081f1c509" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "1X47WVGmwVA=", + "parentSpanId": "XoCVQvOw3eM=", + "flags": 768, + "name": "attachment publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1795164306352857288", + "endTimeUnixNano": "1795164306353855758", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachment-jobs" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "c474b80aef4f42b1ba83aa52fc227420" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "jpUcOdrK9v8=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306353943301", + "endTimeUnixNano": "1795164306358874754", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "uqFoOGdo/j8=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306361469707", + "endTimeUnixNano": "1795164306363607190", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "Djfujh2+aLk=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.get_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306363643778", + "endTimeUnixNano": "1795164306365315189", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "get_object" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "UqSD5a1Hm0M=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306502165196", + "endTimeUnixNano": "1795164306504695870", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ClientError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "FIWbTD1UmKU=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306504774140", + "endTimeUnixNano": "1795164306509272099", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "put_object" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "sGmfWvZhAhE=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306509328359", + "endTimeUnixNano": "1795164306510325785", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ClientError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "cTvaoR1LX60=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306510386972", + "endTimeUnixNano": "1795164306512247503", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "put_object" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "n7+0IIgeeOo=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306512289102", + "endTimeUnixNano": "1795164306513273664", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ClientError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "z5ewZlRmlSI=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306513330090", + "endTimeUnixNano": "1795164306515021299", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "put_object" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "ThQkryD54tU=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.head_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306515051329", + "endTimeUnixNano": "1795164306515871498", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "head_object" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "ClientError" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "3F5tg5VjfcM=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306515917149", + "endTimeUnixNano": "1795164306517590816", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "put_object" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "jRNKytkgcSc=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306517629032", + "endTimeUnixNano": "1795164306523308482", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "EPrL86X8HKM=", + "parentSpanId": "wUZSkS75RVs=", + "flags": 256, + "name": "s3.delete_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306523346489", + "endTimeUnixNano": "1795164306524869630", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "delete_object" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "wUZSkS75RVs=", + "parentSpanId": "1X47WVGmwVA=", + "flags": 768, + "name": "attachment process", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1795164306353917740", + "endTimeUnixNano": "1795164306525244943", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachment-jobs" + } + }, + { + "key": "job.id", + "value": { + "stringValue": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.36.0" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab6.1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "067e0d5099614580ab61516259b76c14" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "s72qs5x19wo=", + "parentSpanId": "bWCElBq0RoM=", + "flags": 256, + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306279538296", + "endTimeUnixNano": "1795164306279912941", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "9hTiABG5UiA=", + "parentSpanId": "bWCElBq0RoM=", + "flags": 256, + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306279948610", + "endTimeUnixNano": "1795164306280391627", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "PPdidexZwfM=", + "parentSpanId": "bWCElBq0RoM=", + "flags": 256, + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306280710138", + "endTimeUnixNano": "1795164306281834242", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "PKeSMnEZomk=", + "parentSpanId": "XoCVQvOw3eM=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306282049715", + "endTimeUnixNano": "1795164306285026998", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "aws-api" + } + }, + { + "key": "rpc.service", + "value": { + "stringValue": "S3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "put_object" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "XoCVQvOw3eM=", + "parentSpanId": "bWCElBq0RoM=", + "flags": 256, + "name": "db.transaction", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795164306281855668", + "endTimeUnixNano": "1795164306285749976", + "attributes": [ + { + "key": "db.system", + "value": { + "stringValue": "postgresql" + } + } + ], + "status": {} + }, + { + "traceId": "pjCRI5a2QQGyk8wwbuRO2A==", + "spanId": "bWCElBq0RoM=", + "parentSpanId": "ASNFZ4mrze8=", + "flags": 768, + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795164306279505300", + "endTimeUnixNano": "1795164306285992974", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + } + ] + } + ] + } + ] + }, + "logs": [ + { + "timestamp": "2026-11-20T08:45:06.285760+00:00", + "severity": "INFO", + "service.name": "messenger-api", + "event": "attachment_accepted", + "sampled": true, + "trace_id": "a630912396b64101b293cc306ee44ed8", + "span_id": "6d6084941ab44683", + "attachment_id": "a6e6251c-5774-4a3b-8424-c72d9568e4a6", + "job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2" + }, + { + "timestamp": "2026-11-20T08:45:06.525174+00:00", + "severity": "INFO", + "service.name": "messenger-worker", + "event": "job_processed", + "sampled": true, + "trace_id": "a630912396b64101b293cc306ee44ed8", + "span_id": "c14652912ef9455b", + "job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2", + "attempt": 1, + "outcome": "ready" + }, + { + "timestamp": "2026-11-20T08:45:06.353780+00:00", + "severity": "INFO", + "service.name": "messenger-dispatcher", + "event": "job_published", + "sampled": true, + "trace_id": "a630912396b64101b293cc306ee44ed8", + "span_id": "d57e3b5951a6c150", + "job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2", + "attempt": 1 + }, + { + "timestamp": "2026-11-20T08:45:06.285952+00:00", + "severity": "INFO", + "service.name": "messenger-api", + "event": "http_request", + "sampled": true, + "trace_id": "a630912396b64101b293cc306ee44ed8", + "span_id": "6d6084941ab44683", + "request_id": "cb06618bb089444e8958f45beb2a8d1f", + "method": "POST", + "route": "/api/v1/chats/{chat_id}/attachments", + "status": 202, + "duration_seconds": 0.006511 + }, + { + "timestamp": "2026-11-20T08:45:06.504597+00:00", + "severity": "INFO", + "service.name": "messenger-worker", + "event": "operation_failed", + "sampled": true, + "trace_id": "a630912396b64101b293cc306ee44ed8", + "span_id": "52a483e5ad479b43", + "error_type": "ClientError" + }, + { + "timestamp": "2026-11-20T08:45:06.510258+00:00", + "severity": "INFO", + "service.name": "messenger-worker", + "event": "operation_failed", + "sampled": true, + "trace_id": "a630912396b64101b293cc306ee44ed8", + "span_id": "b0699f5af6610211", + "error_type": "ClientError" + }, + { + "timestamp": "2026-11-20T08:45:06.513218+00:00", + "severity": "INFO", + "service.name": "messenger-worker", + "event": "operation_failed", + "sampled": true, + "trace_id": "a630912396b64101b293cc306ee44ed8", + "span_id": "9fbfb420881e78ea", + "error_type": "ClientError" + }, + { + "timestamp": "2026-11-20T08:45:06.515824+00:00", + "severity": "INFO", + "service.name": "messenger-worker", + "event": "operation_failed", + "sampled": true, + "trace_id": "a630912396b64101b293cc306ee44ed8", + "span_id": "4e1424af20f9e2d5", + "error_type": "ClientError" + } + ] +} diff --git a/evidence/trace-ui.jpg b/evidence/trace-ui.jpg new file mode 100644 index 0000000..b5ff3eb Binary files /dev/null and b/evidence/trace-ui.jpg differ diff --git a/infra/RUNBOOK.md b/infra/RUNBOOK.md new file mode 100644 index 0000000..e5bd1e6 --- /dev/null +++ b/infra/RUNBOOK.md @@ -0,0 +1,67 @@ +# Локальный runbook — lab6 + +Grafana: http://127.0.0.1:13006/d/messenger-lab6. +Prometheus: http://127.0.0.1:19096. +Loki и Tempo доступны через Explore и provisioned datasources. +Не запрашивать credentials, HTTP bodies, raw URL или SQL parameters для диагностики. + +## api-errors + +1. Открыть API throughput, 5xx ratio и p95/p99. Health исключён. +2. Проверить APIHighErrorRate в Prometheus /alerts: 5xx / все обработанные business requests за 30s > 5%, for 6s. +3. В Loki: {service_name="messenger-api"} | json | status >= 500. +4. Взять trace_id из лога и перейти в Tempo. X-Request-Id связывает ответ с тем же логом. +5. Если основное время/ошибка в db.transaction, проверить db_pool_used/waiting и QueueObservationUnavailable: + проблема на границе PostgreSQL/пула, а не декодирования изображения. +6. Контролируемый инцидент теста удерживает ACCESS EXCLUSIVE lock таблицы sessions: lock_timeout=1.5s даёт 503. + SELECT 1 readiness остаётся доступным; долгий span db.transaction показывает причину, даже если HTTP health зелёный. +7. После снятия блокировки тот же Bearer снова работает. Подождать очистки 30s окна и for; проверить resolved. + +## queue-lag + +1. Проверить queue_depth и queue_oldest_age_seconds: это принятые незавершённые jobs в PostgreSQL, + включая ещё не опубликованный outbox; это не длина Redis stream и не длительность resize. +2. Если upload остаётся 202, а возраст/глубина растут, проверить worker busy/capacity и отсутствие новых job_processed. + Удалённый процесс может оставаться в scrape cache до 30s; учитывать эту задержку. +3. В Loki: {service_name=~"messenger-.*"} | json | event=~"attachment_accepted|job_published|job_processed|job_retry". +4. Через trace_id пройти API → producer → consumer → S3/DB. + Published без consumer указывает на отсутствие/недоступность worker или его чтения broker. + Consumer с retry и ошибочным S3 span указывает на storage, а не потерю задания. +5. Восстановить worker. Проверить ready, уменьшение backlog до нуля, исчезновение QueueLag. +6. Не удалять volume, stream, job registry или staging ради зелёного dashboard. + +## telemetry-backend + +1. Если business API продолжает отвечать, но trace не находится, проверить Collector: + otelcol_exporter_send_failed_spans_total, otelcol_exporter_queue_size/capacity. +2. Уточнить, принимает ли Collector OTLP (otelcol_receiver_accepted_spans_total). + Ошибки exporter при растущем accepted означают проблему downstream, а не отсутствие инструментации. +3. Восстановить Tempo/Loki; проверить новый trace/log. Старые telemetry, отброшенные после 10s retry budget, + не обязаны появиться. Бизнес-данные и telemetry имеют разные гарантии сохранности. +4. Collector: memory limiter 192MiB, batch <=256, две exporter-очереди по 128 batches; + SDK: span queue512/batch128, logs256/batch64, OTLP timeout1s. + Нет бесконечной очереди и обязательной блокирующей синхронной отправки на HTTP-path. +5. У каждого backend именованный volume. Отказ процесса и потеря диска — разные инциденты. + +## SLI / SLO + +Учебное окно 5 минут, при наличии запросов: +- API availability SLI = 1 - sum(increase(http_server_requests_total{status_class="5xx"}[5m])) + / sum(increase(http_server_requests_total[5m])). Цель >=99%. + 4xx считаются корректно обслуженными запросами, health исключён. + Это SLI обработанных API запросов: отказ LB/сети до API в знаменатель не попадает. +- Time-to-ready latency SLI среди завершённых ready jobs: + sum(increase(job_ready_duration_seconds_bucket{le="5"}[5m])) + / sum(increase(job_ready_duration_seconds_count[5m])). Цель >=95%. + Не завершённые/failed jobs не скрываются: отдельно jobs_processed outcomes, job_retries и QueueLag. + Этот latency SLO не является SLO доступности всего pipeline. + +Короткие алерты 30s и 10s queue age предназначены для локальной демонстрации. +Они не заявлены промышленными burn-rate alerts; выбран трек 1. + + +Balanced default: TRACE_SAMPLE_RATIO=0.25; metrics/logs всегда включены. +У несэмплированного запроса trace_id присутствует в log, но полного trace в Tempo не будет. +Для контролируемого proof используется W3C sampled=1; для полной локальной диагностики: +TRACE_SAMPLE_RATIO=1 docker compose up -d --no-deps api worker dispatcher. +Это не tail sampling: ошибки и медленные spans не получают автоматического приоритета. diff --git a/infra/alerts.yaml b/infra/alerts.yaml new file mode 100644 index 0000000..39a6f1f --- /dev/null +++ b/infra/alerts.yaml @@ -0,0 +1,27 @@ +groups: + - name: messenger + rules: + - alert: APIHighErrorRate + expr: sum(rate(http_server_requests_total{service_name="messenger-api",status_class="5xx"}[30s])) / clamp_min(sum(rate(http_server_requests_total{service_name="messenger-api"}[30s])), 0.001) > 0.05 + for: 6s + labels: + severity: warning + annotations: + summary: API 5xx share exceeds 5 percent in the last 30 seconds + runbook: infra/RUNBOOK.md#api-errors + - alert: QueueLag + expr: max(queue_oldest_age_seconds{service_name="messenger-dispatcher"}) > 10 and max(queue_depth{service_name="messenger-dispatcher"}) > 0 + for: 6s + labels: + severity: warning + annotations: + summary: Accepted unfinished image work older than 10 seconds + runbook: infra/RUNBOOK.md#queue-lag + - alert: QueueObservationUnavailable + expr: min(queue_observation_ok{service_name="messenger-dispatcher"}) == 0 + for: 6s + labels: + severity: warning + annotations: + summary: Dispatcher cannot observe PostgreSQL backlog + runbook: infra/RUNBOOK.md#api-errors diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..f57b7c7 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,713 @@ +{ + "uid": "messenger-lab6", + "title": "MTUSI lab6 — Messenger observability", + "schemaVersion": 41, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "tags": [ + "mtusi", + "lab6" + ], + "annotations": { + "list": [] + }, + "panels": [ + { + "id": 1, + "title": "API throughput (health excluded)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(http_server_requests_total{service_name=\"messenger-api\"}[1m]))", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 2, + "title": "API 5xx ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(http_server_requests_total{service_name=\"messenger-api\",status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{service_name=\"messenger-api\"}[1m])),0.001)", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 3, + "title": "API p50 from histogram", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 4, + "title": "API p95 from histogram", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 5, + "title": "API p99 from histogram", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 6, + "title": "Worker jobs by outcome", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (rate(jobs_processed_total{service_name=\"messenger-worker\"}[1m]))", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 7, + "title": "Worker processing p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 8, + "title": "Accepted to ready p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by(le)(rate(job_ready_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 9, + "title": "Accepted unfinished jobs", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "max(queue_depth{service_name=\"messenger-dispatcher\"})", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 10, + "title": "Oldest unfinished job age", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "max(queue_oldest_age_seconds{service_name=\"messenger-dispatcher\"})", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 11, + "title": "DB pool used", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name) (db_pool_used{service_name=\"messenger-api\"})", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 12, + "title": "DB pool waiting", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name) (db_pool_waiting{service_name=\"messenger-api\"})", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 13, + "title": "Worker busy / capacity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 14, + "title": "Process resident memory", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "process_resident_memory_bytes{service_name=~\"messenger-.*\"}", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 15, + "title": "Process CPU seconds per second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name)(rate(process_cpu_seconds_total{service_name=~\"messenger-.*\"}[1m]))", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 16, + "title": "Container memory incl renderer", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 56, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "container_memory_bytes{service_name=~\"messenger-.*\"}", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 17, + "title": "Local alerts", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 64, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertstate=\"firing\"}", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 18, + "title": "Collector failed span exports", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 64, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))", + "legendFormat": "{{service_name}} {{status_class}} {{outcome}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 100, + "title": "Correlated structured logs: open TraceID for API / publish / worker chain", + "type": "logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 0, + "y": 72, + "w": 24, + "h": 10 + }, + "targets": [ + { + "refId": "A", + "expr": "{service_name=~\"messenger-.*\"} | json", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "sortOrder": "Descending" + } + }, + { + "id": 101, + "title": "Terminal failed jobs", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 82, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(job_retries_total{service_name=\"messenger-worker\",outcome=\"failed\"}[1m]))" + } + ] + }, + { + "id": 102, + "title": "Retry attempts", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 82, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(job_retries_total{service_name=\"messenger-worker\",outcome=\"retry\"}[1m]))" + } + ] + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/provider.yaml b/infra/grafana/provisioning/dashboards/provider.yaml new file mode 100644 index 0000000..f24ab58 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/provider.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +providers: + - name: Messenger + folder: MTUSI + type: file + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/sources.yaml b/infra/grafana/provisioning/datasources/sources.yaml new file mode 100644 index 0000000..1c00fbd --- /dev/null +++ b/infra/grafana/provisioning/datasources/sources.yaml @@ -0,0 +1,32 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + datasourceUid: tempo + matcherRegex: '"trace_id":"([a-f0-9]{32})"' + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: '-1m' + spanEndTimeShift: '1m' + filterByTraceID: false + customQuery: true + query: '{service_name=~"messenger-.*"} | json | trace_id="$${__trace.traceId}"' diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..5495e1e --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,46 @@ +global + maxconn 1024 + nbthread 1 + hard-stop-after 10s + +defaults + mode http + timeout connect 500ms + timeout check 500ms + timeout queue 1s + timeout client 30s + timeout server 30s + timeout http-request 3s + retries 2 + retry-on conn-failure + option http-server-close + +resolvers docker + nameserver docker_dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold nx 1s + hold refused 1s + hold timeout 1s + hold other 1s + hold obsolete 1s + +frontend public + bind :8080 + default_backend replicas + +backend replicas + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fall 2 rise 2 + server-template api 1-4 api:8080 check resolvers docker init-addr none resolve-prefer ipv4 + +listen metrics + bind :8404 + stats enable + stats uri /stats diff --git a/infra/loki.yaml b/infra/loki.yaml new file mode 100644 index 0000000..244cca6 --- /dev/null +++ b/infra/loki.yaml @@ -0,0 +1,33 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 24h + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 8 +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem +analytics: + reporting_enabled: false diff --git a/infra/otel-collector.yaml b/infra/otel-collector.yaml new file mode 100644 index 0000000..eea2b42 --- /dev/null +++ b/infra/otel-collector.yaml @@ -0,0 +1,66 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 32 + batch: + timeout: 1s + send_batch_size: 128 + send_batch_max_size: 256 +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + metric_expiration: 30s + resource_to_telemetry_conversion: + enabled: true + otlphttp/tempo: + endpoint: http://tempo:4318 + timeout: 2s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 128 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 2s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 128 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] diff --git a/infra/prometheus.yaml b/infra/prometheus.yaml new file mode 100644 index 0000000..e2f4673 --- /dev/null +++ b/infra/prometheus.yaml @@ -0,0 +1,11 @@ +global: + scrape_interval: 2s + evaluation_interval: 2s +rule_files: [/etc/prometheus/alerts.yaml] +scrape_configs: + - job_name: messenger + static_configs: + - targets: [otel-collector:8889] + - job_name: collector + static_configs: + - targets: [otel-collector:8888] diff --git a/infra/tempo.yaml b/infra/tempo.yaml new file mode 100644 index 0000000..1392b9b --- /dev/null +++ b/infra/tempo.yaml @@ -0,0 +1,24 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +ingester: + trace_idle_period: 5s + max_block_duration: 1m + max_block_bytes: 1048576 +compactor: + compaction: + block_retention: 24h +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +usage_report: + reporting_enabled: false diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..eeac3c1 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,42 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY + , username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$') + , display_name TEXT NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100) + , password_hash TEXT NOT NULL + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +) +; +CREATE TABLE chats ( + id UUID PRIMARY KEY + , title TEXT NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100) + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , last_message_at TIMESTAMPTZ +) +; +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) + , user_id UUID NOT NULL REFERENCES users(id) + , PRIMARY KEY (chat_id, user_id) +) +; +CREATE INDEX chat_members_user ON chat_members (user_id, chat_id) +; +CREATE TABLE messages ( + id UUID PRIMARY KEY + , chat_id UUID NOT NULL REFERENCES chats(id) + , sender_id UUID NOT NULL REFERENCES users(id) + , text TEXT NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000) + , created_at TIMESTAMPTZ NOT NULL + , FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +) +; +CREATE INDEX messages_history ON messages (chat_id, created_at, id) +; +CREATE TABLE sessions ( + token_hash TEXT PRIMARY KEY + , user_id UUID NOT NULL REFERENCES users(id) + , expires_at TIMESTAMPTZ NOT NULL +) +; +CREATE INDEX sessions_expiry ON sessions (expires_at) +; diff --git a/migrations/002_unicode_nul.sql b/migrations/002_unicode_nul.sql new file mode 100644 index 0000000..224cda0 --- /dev/null +++ b/migrations/002_unicode_nul.sql @@ -0,0 +1,17 @@ +-- PostgreSQL TEXT cannot contain U+0000. Store validated UTF-8 losslessly in BYTEA. +-- API validation keeps the contract's code-point limits; these bounds cap bytes. +ALTER TABLE users + DROP CONSTRAINT users_display_name_check + , ALTER COLUMN display_name TYPE BYTEA USING convert_to(display_name, 'UTF8') + , ADD CONSTRAINT users_display_name_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400) +; +ALTER TABLE chats + DROP CONSTRAINT chats_title_check + , ALTER COLUMN title TYPE BYTEA USING convert_to(title, 'UTF8') + , ADD CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400) +; +ALTER TABLE messages + DROP CONSTRAINT messages_text_check + , ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8') + , ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000) +; diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 0000000..3ca53a7 --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) + , chat_id UUID NOT NULL REFERENCES chats(id) + , key VARCHAR(128) NOT NULL + , request_hash CHAR(64) NOT NULL + , message_id UUID REFERENCES messages(id) + , expires_at TIMESTAMPTZ NOT NULL + , PRIMARY KEY (user_id, chat_id, key) +) +; +CREATE INDEX idempotency_expiry ON idempotency_keys (expires_at) +; diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..bfd5189 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,30 @@ +ALTER TABLE messages + DROP CONSTRAINT messages_text_bytes + , ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000) + , ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}' +; +CREATE TABLE attachments ( + id UUID PRIMARY KEY + , chat_id UUID NOT NULL REFERENCES chats(id) + , owner_id UUID NOT NULL REFERENCES users(id) + , filename TEXT NOT NULL + , object_key TEXT NOT NULL UNIQUE + , content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')) + , size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760) + , sha256 CHAR(64) NOT NULL + , state TEXT NOT NULL CHECK (state IN ('uploading', 'ready', 'storage_missing')) + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id) +) +; +CREATE INDEX attachments_chat ON attachments (chat_id, id) +; +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) + , attachment_id UUID NOT NULL REFERENCES attachments(id) + , PRIMARY KEY (message_id, attachment_id) +) +; +CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id) +; diff --git a/migrations/005_pipeline.sql b/migrations/005_pipeline.sql new file mode 100644 index 0000000..0ef1e81 --- /dev/null +++ b/migrations/005_pipeline.sql @@ -0,0 +1,40 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_state_check + , ADD CONSTRAINT attachments_state_check CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed', 'storage_missing')) + , ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1 + , ADD COLUMN staging_key TEXT + , ADD COLUMN variants JSONB NOT NULL DEFAULT '[]' + , ADD COLUMN error_code TEXT + , ADD COLUMN current_job_id UUID +; +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY + , attachment_id UUID NOT NULL REFERENCES attachments(id) + , pipeline_version INTEGER NOT NULL + , source_key TEXT NOT NULL + , state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded')) + , attempt INTEGER NOT NULL DEFAULT 0 + , max_attempts INTEGER NOT NULL DEFAULT 5 + , lease_token UUID + , lease_until TIMESTAMPTZ + , next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , published_at TIMESTAMPTZ + , error_code TEXT + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , UNIQUE (attachment_id, pipeline_version) +) +; +ALTER TABLE attachments + ADD CONSTRAINT attachments_current_job FOREIGN KEY (current_job_id) REFERENCES attachment_jobs(id) +; +CREATE INDEX jobs_due ON attachment_jobs (state, next_attempt_at) +; +CREATE TABLE job_events ( + id BIGSERIAL PRIMARY KEY + , job_id UUID NOT NULL REFERENCES attachment_jobs(id) + , event TEXT NOT NULL + , attempt INTEGER NOT NULL + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +) +; diff --git a/migrations/006_observability.sql b/migrations/006_observability.sql new file mode 100644 index 0000000..e19783d --- /dev/null +++ b/migrations/006_observability.sql @@ -0,0 +1,3 @@ +ALTER TABLE attachment_jobs + ADD COLUMN trace_context JSONB NOT NULL DEFAULT '{}' +; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7fa5df3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,27 @@ +boto3==1.40.61 +botocore==1.40.76 +certifi==2026.10.23 +charset-normalizer==3.5.2 +googleapis-common-protos==1.76.0 +idna==3.20 +importlib_metadata==8.7.1 +jmespath==1.1.0 +opentelemetry-api==1.36.0 +opentelemetry-exporter-otlp-proto-common==1.36.0 +opentelemetry-exporter-otlp-proto-http==1.36.0 +opentelemetry-proto==1.36.0 +opentelemetry-sdk==1.36.0 +opentelemetry-semantic-conventions==0.57b0 +pillow==13.0.0 +protobuf==6.33.6 +psycopg==3.2.3 +psycopg-binary==3.2.3 +psycopg-pool==3.2.4 +python-dateutil==2.9.0.post0 +redis==5.2.1 +requests==2.34.2 +s3transfer==0.14.0 +six==1.17.0 +typing_extensions==4.16.0 +urllib3==2.8.0 +zipp==4.1.2 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 0000000..4b118cb --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Create local credentials without printing or overwriting existing values.""" +import json +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] +directory = root / 'secrets' +directory.mkdir(mode=0o700, exist_ok=True) + +def save(name, value): + path = directory / name + if not path.exists(): + path.write_text(value) + path.chmod(0o644) # UID 10001 in Compose; parent is owner-only 0700. + return path.read_text().strip() + +password = save('postgres_password', secrets.token_hex(24)) +save('database_url', 'postgresql://messenger:' + password + '@postgres:5432/messenger') +save('cache_url', 'redis://redis:6379/0') +save('broker_url', 'redis://broker:6379/0') +identities = [] +for role, actions in { + 'admin': ['Admin', 'Read', 'Write', 'List'], + 'api': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'], + 'worker': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'], +}.items(): + key = save('s3_' + role + '_key', secrets.token_hex(12)) + secret = save('s3_' + role + '_secret', secrets.token_hex(24)) + identities.append({'name': role, 'credentials': [{'accessKey': key, 'secretKey': secret}], 'actions': actions}) +configuration = directory / 's3_config' +configuration.write_text(json.dumps({'identities': identities})) +configuration.chmod(0o600) # Only the root S3 gateway mounts the full identity configuration. +env = root / '.env' +if not env.exists(): + env.write_text((root / '.env.example').read_text()) + env.chmod(0o600) +print('Local configuration ready; no secrets printed.') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..dfde59d --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/snapshot.py > evidence/restart-before.json +docker compose stop api lb worker dispatcher +docker compose restart postgres redis s3-store s3 broker otel-collector prometheus loki tempo grafana +docker compose up -d --wait postgres redis s3-store s3 broker otel-collector prometheus loki tempo grafana +docker compose up -d --wait api lb worker dispatcher +python3 scripts/snapshot.py > evidence/restart-after.json diff --git a/scripts/snapshot.py b/scripts/snapshot.py new file mode 100644 index 0000000..3d0354f --- /dev/null +++ b/scripts/snapshot.py @@ -0,0 +1,11 @@ +"""Sanitized process and resource evidence; never dump container environments.""" +import json +import subprocess + +ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split() +records = [] +for container in ids: + raw = subprocess.check_output(['docker', 'inspect', container], text=True) + data = json.loads(raw)[0] + records.append({'container': data['Id'], 'name': data['Name'], 'service': data['Config']['Labels']['com.docker.compose.service'], 'image': data['Config']['Image'], 'user': data['Config']['User'], 'running': data['State']['Running'], 'started_at': data['State']['StartedAt'], 'exit_code': data['State']['ExitCode'], 'oom_killed': data['State']['OOMKilled'], 'memory_limit': data['HostConfig']['Memory'], 'nano_cpus': data['HostConfig']['NanoCpus'], 'ports': data['NetworkSettings']['Ports'], 'mounts': [{'type': m['Type'], 'destination': m['Destination'], 'name': m.get('Name')} for m in data['Mounts']]}) +print(json.dumps(records, indent=2)) diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..ce9279f --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/stop_one.py diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 0000000..bc495aa --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,32 @@ +"""Resolve the requested process alias using each actual container, never a fake header.""" +import json +import os +from pathlib import Path +import subprocess +import time + + +def docker(*args): + return subprocess.check_output(['docker', *args], text=True).strip() + + +def instance(container): + code = "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=1).headers['X-Instance-Id'])" + return docker('exec', container, 'python', '-c', code) + + +if __name__ == '__main__': + target = os.environ['TARGET_INSTANCE_ID'] + started = time.monotonic() + for container in docker('compose', 'ps', '-q', 'api').splitlines(): + if instance(container) == target: + docker('stop', '--time', '8', container) + state = json.loads(docker('inspect', '--format', '{{json .State}}', container)) + assert not state['Running'], 'target must remain stopped' + record = {'instance': target, 'container': container, 'running': state['Running'], 'exit_code': state['ExitCode'], 'seconds': round(time.monotonic() - started, 3)} + with Path('evidence/stopped-instances.jsonl').open('a') as out: + out.write(json.dumps(record) + '\n') + print(json.dumps(record)) + break + else: + raise SystemExit('Requested live API instance not found') diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 0000000..e635cae --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,16 @@ +import os +import time +import urllib.request + +base = os.environ.get('BASE_URL', 'http://127.0.0.1:18087') +for attempt in range(120): + try: + with urllib.request.urlopen(base + '/health/ready', timeout=2) as response: + if response.status == 200: + print('Ready through load balancer.') + break + except OSError: + pass + time.sleep(0.5) +else: + raise SystemExit('Readiness timeout') diff --git a/src/admin.py b/src/admin.py new file mode 100644 index 0000000..699bd3e --- /dev/null +++ b/src/admin.py @@ -0,0 +1,127 @@ +"""Explicit local operator tools; no administrative HTTP endpoint.""" +import argparse +import json +import os +import uuid + +from pipeline import connect, audit +from objects import Objects +from postgres import setting + + +def replay(job_id): + with connect() as conn, conn.transaction(): + job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job_id,)).fetchone() + if job is None or job['state'] != 'failed': + raise ValueError('Replay requires a failed job') + attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND current_job_id = %s FOR UPDATE', (job_id,)).fetchone() + if attachment is None or attachment['state'] != 'failed' or Objects(setting).head(job['source_key']) is None: + raise ValueError('Current failed source is unavailable') + conn.execute(''' + UPDATE attachment_jobs + SET state = 'queued' + , max_attempts = attempt + %s + , published_at = NULL + , next_attempt_at = clock_timestamp() + , error_code = NULL + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + ''', (int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), job_id)) + conn.execute("UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job_id,)) + audit(conn, job, 'operator_replay') + return {'job_id': str(job_id), 'replayed': True} + + +def version(attachment_id, number): + if number not in (1, 2): + raise ValueError('Only implemented pipeline versions 1 and 2 are supported') + with connect() as conn, conn.transaction(): + row = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() + if row is None or number != row['pipeline_version'] + 1 or row['state'] in ('uploading', 'failed', 'storage_missing'): + raise ValueError('Version must advance the current process exactly once') + old = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (row['current_job_id'],)).fetchone() + source = row['object_key'] if row['state'] == 'ready' else old['source_key'] + if Objects(setting).head(source) is None: + raise ValueError('Source unavailable') + conn.execute("UPDATE attachment_jobs SET state = 'superseded', lease_token = NULL, lease_until = NULL WHERE 1=1 AND id = %s", (old['id'],)) + audit(conn, old, 'operator_supersede') + job_id = uuid.uuid4() + conn.execute(''' + INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts) + VALUES (%s, %s, %s, %s, %s) + ''', (job_id, attachment_id, number, source, int(os.environ.get('JOB_MAX_ATTEMPTS', '5')))) + conn.execute(''' + UPDATE attachments + SET pipeline_version = %s + , current_job_id = %s + , state = 'queued' + , variants = '[]' + , error_code = NULL + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + ''', (number, job_id, attachment_id)) + return {'job_id': str(job_id), 'version': number} + + +def cleanup(apply=False, retention=86400): + if retention < 86400: + raise ValueError('Retention must be >=24 hours') + objects = Objects(setting) + counts = {'ready_staging': 0, 'expired_staging': 0} + with connect() as conn: + # Intent rows cover every valid staging PUT. Never collect active jobs. + last = '00000000-0000-0000-0000-000000000000' + while True: + rows = conn.execute('SELECT id FROM attachments WHERE 1=1 AND id > %s ORDER BY id LIMIT 100', (last,)).fetchall() + if not rows: + break + for candidate in rows: + last = candidate['id'] + with conn.transaction(): + row = conn.execute(''' + SELECT * + FROM attachments + WHERE 1=1 + AND id = %s + AND (state = 'ready' OR (state IN ('uploading', 'failed') AND updated_at < clock_timestamp() - %s * INTERVAL '1 second')) + FOR UPDATE SKIP LOCKED + ''', (last, retention)).fetchone() + if row is None or not row['staging_key'] or not row['staging_key'].startswith('staging/'): + continue + counts['ready_staging' if row['state'] == 'ready' else 'expired_staging'] += 1 + if apply: + objects.delete(row['staging_key']) + if row['state'] == 'uploading' and row['current_job_id'] is None: + conn.execute('DELETE FROM attachments WHERE 1=1 AND id = %s', (last,)) + else: + conn.execute('UPDATE attachments SET staging_key = NULL WHERE 1=1 AND id = %s', (last,)) + return dict(apply=apply, retention_seconds=retention, counts=counts) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('action', choices=['failed', 'replay', 'version', 'cleanup']) + p.add_argument('identifier', nargs='?') + p.add_argument('--version', type=int) + p.add_argument('--apply', action='store_true') + args = p.parse_args() + try: + if args.action == 'failed': + with connect() as conn: + result = conn.execute("SELECT id, attachment_id, pipeline_version, attempt, max_attempts, error_code FROM attachment_jobs WHERE 1=1 AND state = 'failed' ORDER BY created_at LIMIT 100").fetchall() + elif args.action == 'replay': + result = replay(uuid.UUID(args.identifier)) + elif args.action == 'version': + result = version(uuid.UUID(args.identifier), args.version) + else: + result = cleanup(args.apply) + print(json.dumps(result, default=str, indent=2)) + except Exception: + print(json.dumps({'error': 'Operator action refused or dependency unavailable; no secrets logged.'})) + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/src/bucket_bootstrap.py b/src/bucket_bootstrap.py new file mode 100644 index 0000000..a0a2cbd --- /dev/null +++ b/src/bucket_bootstrap.py @@ -0,0 +1,13 @@ +"""Idempotent private bucket bootstrap, executed only with administrative S3 keys.""" +from botocore.exceptions import ClientError +from objects import Objects +from postgres import setting + +objects = Objects(setting) +try: + objects.client.create_bucket(Bucket=objects.bucket) +except ClientError as exc: + if exc.response['Error']['Code'] not in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'): + raise RuntimeError('Bucket bootstrap failed') from None +objects.ready() +print('Private bucket ready; static authenticated identities loaded from local secret config.') diff --git a/src/common.py b/src/common.py new file mode 100644 index 0000000..29c87c5 --- /dev/null +++ b/src/common.py @@ -0,0 +1,98 @@ +"""Shared HTTP contract; storage is supplied by each laboratory.""" +import base64 +import datetime as dt +import json +import re +import uuid + + +class APIError(Exception): + def __init__(self, status, code, message): + self.status, self.code, self.message = status, code, message + + +def invalid(): + raise APIError(400, 'invalid_request', 'Неверные параметры запроса') + + +def unauthorized(): + raise APIError(401, 'unauthorized', 'Требуется действительная идентификация') + + +def missing(): + raise APIError(404, 'not_found', 'Объект не найден') + + +def identifier(value): + try: + if not isinstance(value, str): + invalid() + return str(uuid.UUID(value)) + except ValueError: + invalid() + + +def text(value, maximum, minimum=1): + if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip(): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + return value + + +def fields(value, required): + if not isinstance(value, dict) or set(value) != set(required): + invalid() + + +def username(value): + if not isinstance(value, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', value): + invalid() + return value + + +def now(): + return dt.datetime.now(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def page_args(query, scope): + if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()): + invalid() + limit = query.get('limit', ['50'])[0] + if not re.fullmatch(r'[0-9]+', limit): + invalid() + try: + limit = int(limit) + except ValueError: + invalid() + if not 1 <= limit <= 100: + invalid() + cursor = None + if 'cursor' in query: + raw = query['cursor'][0] + try: + if not raw or len(raw) > 1024: + invalid() + decoded = json.loads(base64.b64decode(raw + '=' * (-len(raw) % 4), altchars=b'-_', validate=True)) + if not isinstance(decoded, list) or len(decoded) != 3 or decoded[0] != scope: + invalid() + stamp = decoded[1] + if not isinstance(stamp, str) or not stamp.endswith('Z'): + invalid() + stamp = dt.datetime.fromisoformat(stamp[:-1] + '+00:00').isoformat(timespec='microseconds').replace('+00:00', 'Z') + cursor = (stamp, identifier(decoded[2])) + except (ValueError, TypeError, UnicodeError): + invalid() + return limit, cursor + + +def page(items, limit, scope): + more = len(items) > limit + items = items[:limit] + cursor = None + if more: + last = items[-1] + cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=') + return {'items': items, 'next_cursor': cursor} diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..ebd85f7 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,13 @@ +"""One-shot migration job; replicas never execute DDL.""" +from pathlib import Path +import psycopg +from postgres import setting + +with psycopg.connect(setting('DATABASE_URL'), connect_timeout=5) as conn: + conn.execute('SELECT pg_advisory_xact_lock(2301003)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)') + for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')): + if not conn.execute('SELECT version FROM schema_migrations WHERE 1=1 AND version = %s', (path.name,)).fetchone(): + conn.execute(path.read_text()) + conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.name,)) + print('Applied', path.name) diff --git a/src/objects.py b/src/objects.py new file mode 100644 index 0000000..b4c5749 --- /dev/null +++ b/src/objects.py @@ -0,0 +1,158 @@ +"""Bounded S3 operations. No root credentials or filename-derived object keys.""" +import contextlib +import telemetry as tele +import hashlib +import os +import tempfile +import threading +import time +import unicodedata +import warnings + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, PngImagePlugin, UnidentifiedImageError + +from common import APIError, invalid + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024 +PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024 +warnings.filterwarnings('error', category=Image.DecompressionBombWarning) +UPLOAD_SLOTS = threading.BoundedSemaphore(8) +DECODE_SLOTS = threading.BoundedSemaphore(1) + + +def unavailable(): + raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно') + + +def filename(value): + if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip(): + invalid() + if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + return value + + +@contextlib.contextmanager +def bounded_upload(): + if not UPLOAD_SLOTS.acquire(blocking=False): + raise APIError(503, 'unavailable', 'Все слоты загрузки заняты') + try: + yield + finally: + UPLOAD_SLOTS.release() + + +@contextlib.contextmanager +def receive(handler): + lengths = handler.headers.get_all('Content-Length', []) + if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'): + invalid() + try: + length = int(lengths[0]) + except ValueError: + invalid() + if length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Максимум 10 MiB') + if length <= 0: + invalid() + mime = handler.headers.get_content_type() + if mime not in ('image/png', 'image/jpeg'): + raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG') + deadline = time.monotonic() + 15 + digest = hashlib.sha256() + # Ephemeral bounded spool is never a durable source of a confirmed attachment. + with tempfile.TemporaryFile(mode='w+b') as source: + remaining = length + while remaining: + handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic()))) + if time.monotonic() >= deadline: + invalid() + try: + chunk = handler.rfile.read1(min(64 * 1024, remaining)) + except (OSError, TimeoutError): + invalid() + if not chunk: + invalid() + digest.update(chunk) + source.write(chunk) + remaining -= len(chunk) + if not DECODE_SLOTS.acquire(timeout=10): + raise APIError(503, 'unavailable', 'Проверка изображений перегружена') + try: + source.flush() + for decode in (False,): + source.seek(0) + # Pillow.close releases pixel buffers and may close its input file. + # A duplicate descriptor preserves the original for the S3 PUT. + with os.fdopen(os.dup(source.fileno()), 'rb') as image_file: + image = Image.open(image_file, formats=['PNG', 'JPEG']) + try: + if image.width * image.height > MAX_PIXELS: + raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') + if Image.MIME[image.format] != mime: + raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению') + if decode: + image.load() + else: + image.verify() + finally: + image.close() # Free pixels BEFORE releasing the decoder semaphore. + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None + finally: + DECODE_SLOTS.release() + source.seek(0) + yield source, length, mime, digest.hexdigest() + + +class Objects: + def __init__(self, setting): + self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments') + self.client = boto3.client('s3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:8333'), region_name='us-east-1', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required')) + self.client = tele.S3Client(self.client) + + def head(self, key): + try: + return self.client.head_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'): + return None + unavailable() + except BotoCoreError: + unavailable() + + def ready(self): + try: + self.client.head_bucket(Bucket=self.bucket) + except (ClientError, BotoCoreError): + unavailable() + + def put(self, key, source, length, mime, digest): + try: + self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest}) + except (ClientError, BotoCoreError): + unavailable() + + def get(self, key): + try: + return self.client.get_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError): + unavailable() + + def delete(self, key): + try: + self.client.delete_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError): + unavailable() diff --git a/src/pipeline.py b/src/pipeline.py new file mode 100644 index 0000000..2627a60 --- /dev/null +++ b/src/pipeline.py @@ -0,0 +1,393 @@ +"""Durable outbox, Redis Streams delivery, bounded worker and fenced result commits.""" +import argparse +import concurrent.futures +import contextlib +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import random +import re +import signal +import subprocess +import sys +import tempfile +import threading +import time +import uuid +import telemetry as tele + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +import redis +from botocore.exceptions import ClientError, BotoCoreError +from common import APIError +from objects import Objects, MAX_BYTES +from postgres import setting + +STREAM, GROUP = 'attachment-jobs', 'image-workers' +LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '60')) +STOP = threading.Event() + + +def raw_connect(): + return psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row, autocommit=True, connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1500') + + +@contextlib.contextmanager +def connect(): + with tele.span('db.transaction', {'db.system':'postgresql'}, tele.SpanKind.CLIENT): + with raw_connect() as connection: + yield connection + + +def broker(): + client = redis.Redis.from_url(setting('BROKER_URL'), decode_responses=True, socket_connect_timeout=1, socket_timeout=3) + try: + client.xgroup_create(STREAM, GROUP, id='0', mkstream=True) + except redis.ResponseError as error: + if 'BUSYGROUP' not in str(error): + raise + return client + + +def event(job): + return {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], 'attempt': job['attempt'] + 1, 'created_at': job['created_at'].isoformat().replace('+00:00', 'Z'), **(job.get('trace_context') or {})} + + +def validate(message): + required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'} + if not isinstance(message, dict) or set(message) - required - {'traceparent', 'tracestate'} or not required <= set(message): + return False + try: + uuid.UUID(message['job_id']); uuid.UUID(message['attachment_id']) + stamp = dt.datetime.fromisoformat(message['created_at'].replace('Z', '+00:00')) + if stamp.tzinfo is None or not message['created_at'].endswith('Z'): + return False + if 'traceparent' in message and (not isinstance(message['traceparent'], str) or not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', message['traceparent'])): + return False + if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512): + return False + return type(message['schema_version']) is int and message['schema_version'] == 1 and message['type'] == 'attachment.process.v1' and all(type(message[k]) is int and message[k] >= 1 for k in ['pipeline_version', 'attempt']) + except (ValueError, TypeError, AttributeError): + return False + + +def audit(conn, job, name): + conn.execute('INSERT INTO job_events (job_id, event, attempt) VALUES (%s, %s, %s)', (job['id'], name, job['attempt'])) + + +def retry(conn, job, code, permanent=False): + failed = permanent or job['attempt'] >= job['max_attempts'] + state = 'failed' if failed else 'queued' + code = code if permanent or not failed else 'retry_exhausted' + delay = min(8, .5 * 2 ** min(job['attempt'] - 1, 4)) + random.uniform(0, .5) + conn.execute(''' + UPDATE attachment_jobs + SET state = %s + , error_code = %s + , lease_token = NULL + , lease_until = NULL + , published_at = NULL + , next_attempt_at = clock_timestamp() + %s * INTERVAL '1 second' + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + ''', (state, code, delay, job['id'])) + conn.execute(''' + UPDATE attachments + SET state = %s + , error_code = %s + , updated_at = clock_timestamp() + WHERE 1=1 + AND current_job_id = %s + ''', (state, code if failed else None, job['id'])) + audit(conn, job, 'failed:' + code if failed else 'retry:' + code) + tele.count('job_retries', {'job_type':'image', 'outcome':'failed' if failed else 'retry'}) + tele.log('job_failed' if failed else 'job_retry', job_id=str(job['id']), attempt=job['attempt'], error_code=code) + + +def dispatch_once(client, fault=None): + with connect() as conn: + with conn.transaction(): + expired = conn.execute("SELECT * FROM attachment_jobs WHERE 1=1 AND state = 'processing' AND lease_until < clock_timestamp() FOR UPDATE SKIP LOCKED LIMIT 20").fetchall() + for job in expired: + retry(conn, job, 'lease_expired') + with conn.transaction(): + jobs = conn.execute(''' + SELECT j.* + FROM attachment_jobs j + JOIN attachments a ON a.current_job_id = j.id + WHERE 1=1 + AND j.state = 'queued' + AND j.next_attempt_at <= clock_timestamp() + AND (j.published_at IS NULL OR j.published_at < clock_timestamp() - %s * INTERVAL '1 second') + ORDER BY j.created_at + LIMIT 10 + FOR UPDATE OF j SKIP LOCKED + ''', (LEASE,)).fetchall() + for job in jobs: + with tele.span('attachment publish', {'messaging.system':'redis', 'messaging.destination.name':STREAM}, + tele.SpanKind.PRODUCER, job.get('trace_context') or {}): + message = event(job) + message.update(tele.carrier()) + client.xadd(STREAM, {'event': json.dumps(message, separators=(',', ':'))}) + tele.log('job_published', job_id=str(job['id']), attempt=job['attempt']+1) + if fault: + fault('after_publish', job) + conn.execute('UPDATE attachment_jobs SET published_at = clock_timestamp() WHERE 1=1 AND id = %s', (job['id'],)) + audit(conn, job, 'published') + return len(jobs) + + +def claim(message): + with connect() as conn, conn.transaction(): + job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (message['job_id'],)).fetchone() + if job is None: + return None + attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (job['attachment_id'],)).fetchone() + if str(job['attachment_id']) != message['attachment_id'] or job['pipeline_version'] != message['pipeline_version']: + return None + if attachment['current_job_id'] != job['id'] or attachment['pipeline_version'] != job['pipeline_version']: + audit(conn, job, 'stale_noop') + return None + if job['state'] in ('ready', 'failed', 'superseded'): + audit(conn, job, 'terminal_noop') + return None + if job['state'] != 'queued' or job['next_attempt_at'] > dt.datetime.now(dt.timezone.utc) or message['attempt'] != job['attempt'] + 1: + audit(conn, job, 'duplicate_or_early_noop') + return None + token = uuid.uuid4() + job = conn.execute(''' + UPDATE attachment_jobs + SET state = 'processing' + , attempt = attempt + 1 + , lease_token = %s + , lease_until = clock_timestamp() + %s * INTERVAL '1 second' + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + RETURNING * + ''', (token, LEASE, job['id'])).fetchone() + conn.execute("UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job['id'],)) + audit(conn, job, 'claimed') + return job, attachment + + +class Permanent(Exception): + pass + + +def immutable(objects, key, path, mime, sha): + head = objects.head(key) + if head is not None: + if head['ContentLength'] != path.stat().st_size or head.get('Metadata', {}).get('sha256') != sha: + raise Permanent('result_conflict') + return + try: + with path.open('rb') as source: + objects.client.put_object(Bucket=objects.bucket, Key=key, Body=source, ContentLength=path.stat().st_size, ContentType=mime, Metadata={'sha256': sha}, IfNoneMatch='*') + except ClientError as error: + if error.response['ResponseMetadata']['HTTPStatusCode'] in (409, 412): + head = objects.head(key) + if head and head['ContentLength'] == path.stat().st_size and head.get('Metadata', {}).get('sha256') == sha: + return + raise + + +def process_core(message, fault=None): + claimed = claim(message) + if claimed is None: + return 'noop' + job, attachment = claimed + objects = Objects(setting) + try: + if fault: + fault('before_side_effect', job) + with tempfile.TemporaryDirectory(prefix='job-') as temporary: + directory = Path(temporary) + source = directory / 'original' + head = objects.head(job['source_key']) + if head is None: + raise Permanent('source_missing') + response = objects.get(job['source_key']) + digest, size = hashlib.sha256(), 0 + deadline = time.monotonic() + 15 + with response['Body'] as stream, source.open('wb') as out: + for chunk in iter(lambda: stream.read(64 * 1024), b''): + size += len(chunk) + if size > MAX_BYTES or time.monotonic() > deadline: + raise Permanent('invalid_image') + digest.update(chunk); out.write(chunk) + if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']: + raise Permanent('checksum_mismatch') + result = subprocess.run([sys.executable, str(Path(__file__).with_name('render.py')), str(source), str(directory), str(job['pipeline_version'])], capture_output=True, timeout=int(os.environ.get('JOB_TIMEOUT_SECONDS', '20'))) + if result.returncode == 2: + raise Permanent('invalid_image') + if result.returncode: + raise RuntimeError('render_failed') + variants = json.loads(result.stdout) + prefix = 'results/' + str(attachment['id']) + '/v' + str(job['pipeline_version']) + '/' + original_key = prefix + 'original' + immutable(objects, original_key, source, attachment['content_type'], attachment['sha256']) + if fault: + fault('after_original', job) + for variant in variants: + variant['object_key'] = prefix + variant['name'] + immutable(objects, variant['object_key'], directory / variant['name'], variant['content_type'], variant['sha256']) + if fault: + fault('after_objects', job) + with connect() as conn, conn.transaction(): + current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone() + changed = conn.execute(''' + UPDATE attachments + SET state = 'ready' + , object_key = %s + , variants = %s + , error_code = NULL + , updated_at = clock_timestamp() + WHERE 1=1 + AND current_job_id = %s + AND pipeline_version = %s + AND %s + RETURNING id + ''', (original_key, Jsonb(variants), job['id'], job['pipeline_version'], current['state'] == 'processing' and current['lease_token'] == job['lease_token'] and current['lease_until'] > dt.datetime.now(dt.timezone.utc))).fetchone() + if changed: + conn.execute("UPDATE attachment_jobs SET state = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE 1=1 AND id = %s", (job['id'],)) + audit(conn, job, 'ready') + else: + audit(conn, job, 'fenced_noop') + if fault: + fault('after_commit', job) + if changed and job['source_key'].startswith('staging/'): + objects.delete(job['source_key']) + if changed: + tele.duration('job_ready_duration', (dt.datetime.now(dt.timezone.utc)-job['created_at']).total_seconds(), {'job_type':'image'}) + return 'ready' if changed else 'fenced' + except Exception as error: + code = str(error) if isinstance(error, Permanent) else 'processing_timeout' if isinstance(error, subprocess.TimeoutExpired) else 'storage_unavailable' if isinstance(error, (APIError, ClientError, BotoCoreError)) else 'processing_failed' + with connect() as conn, conn.transaction(): + current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone() + if current['state'] == 'processing' and current['lease_token'] == job['lease_token']: + retry(conn, current, code, isinstance(error, Permanent)) + return 'failed_or_retry' + finally: + objects.client.close() + + +def process(message, fault=None): + started = time.monotonic() + with tele.span('attachment process', {'messaging.system':'redis', 'messaging.destination.name':STREAM, + 'job.id':message['job_id'], 'job.attempt':message['attempt']}, + tele.SpanKind.CONSUMER, message): + with tele.LOCK: + tele.STATE['worker_busy'] += 1 + outcome = 'error' + try: + outcome = process_core(message, fault) + if outcome == 'failed_or_retry': + tele.trace.get_current_span().set_status(tele.Status(tele.StatusCode.ERROR)) + return outcome + finally: + with tele.LOCK: + tele.STATE['worker_busy'] -= 1 + attributes = {'job_type':'image', 'outcome':outcome} + tele.count('jobs_processed', attributes) + tele.duration('job_processing_duration', time.monotonic()-started, attributes) + tele.log('job_processed', job_id=message['job_id'], attempt=message['attempt'], outcome=outcome) + + +def observe_queue(): + try: + with raw_connect() as conn: + row = conn.execute(""" + SELECT + count(*) AS depth + , coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS age + FROM attachment_jobs + WHERE 1=1 + AND state IN ('queued', 'processing') + """).fetchone() + tele.STATE.update(queue_depth=row['depth'], queue_oldest_age=float(row['age']), queue_observation_ok=1) + except psycopg.Error: + tele.STATE['queue_observation_ok'] = 0 + + +def ack(client, identifier): + # Atomic ack+delete; PostgreSQL is the durable completion/replay registry. + client.eval("redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])", 1, STREAM, GROUP, identifier) + + +def consume(once=False, fault=None): + client = broker() + consumer = uuid.uuid4().hex + while not STOP.is_set(): + messages = [] + try: + pending = client.xautoclaim(STREAM, GROUP, consumer, min_idle_time=LEASE * 1000, start_id='0-0', count=1)[1] + if pending: + messages = pending + else: + batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=500) + messages = batches[0][1] if batches else [] + for identifier, fields in messages: + raw = fields.get('event', '') + try: + message = json.loads(raw) if len(raw) <= 4096 else None + except ValueError: + message = None + if validate(message): + process(message, fault) + else: + client.xadd('rejected-jobs', {'source_message_id': identifier, 'sha256': hashlib.sha256(raw.encode()).hexdigest(), 'reason': 'invalid_envelope'}, maxlen=1000, approximate=False) + ack(client, identifier) + if once: + return + except (redis.RedisError, psycopg.Error): + STOP.wait(.5) + if once and not messages: + return + client.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=['dispatch', 'work']) + parser.add_argument('--once', action='store_true') + parser.add_argument('--fault-point', choices=['before_side_effect', 'after_original', 'after_objects', 'after_commit', 'after_publish']) + args = parser.parse_args() + tele.configure('messenger-dispatcher' if args.mode=='dispatch' else 'messenger-worker') + # Operator-only opt-in injection; no request header or public endpoint controls it. + def fault(point, job): + if point == args.fault_point: + os._exit(77) + signal.signal(signal.SIGTERM, lambda *_: STOP.set()) + signal.signal(signal.SIGINT, lambda *_: STOP.set()) + if args.mode == 'dispatch': + client = broker() + while not STOP.is_set(): + observe_queue() + try: + dispatch_once(client, fault) + except (redis.RedisError, psycopg.Error): + pass + if args.once: + break + STOP.wait(.2) + else: + count = int(os.environ.get('WORKER_CONCURRENCY', '1')) + if not 1 <= count <= 2: + raise ValueError('WORKER_CONCURRENCY must be 1..2') + with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool: + futures = [pool.submit(consume, args.once, fault) for _ in range(count)] + for result in futures: + result.result() + + +if __name__ == '__main__': + try: + main() + finally: + tele.shutdown() diff --git a/src/postgres.py b/src/postgres.py new file mode 100644 index 0000000..7039817 --- /dev/null +++ b/src/postgres.py @@ -0,0 +1,414 @@ +import base64 +import contextlib +import datetime as dt +import hashlib +import hmac +import json +import os +import re +from pathlib import Path +import secrets +import threading +import uuid +import telemetry as tele + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool, PoolTimeout +import redis + +from objects import Objects, unavailable + +from common import APIError, fields, identifier, invalid, missing, page, text, unauthorized, username + +KDF_SLOTS = threading.BoundedSemaphore(2) + + +def setting(name): + path = os.environ.get(name + '_FILE') + return Path(path).read_text().strip() if path else os.environ[name] + + +def public(row): + return {k: (v.decode('utf-8') if isinstance(v, bytes) else [str(x) for x in v] if isinstance(v, list) else str(v) if isinstance(v, uuid.UUID) else v.isoformat(timespec='microseconds').replace('+00:00', 'Z') if isinstance(v, dt.datetime) else v) for k, v in row.items() if k != 'password_hash'} + + +def password_hash(password, salt=None): + salt = salt or secrets.token_bytes(16) + if not KDF_SLOTS.acquire(timeout=1.5): + raise APIError(503, 'unavailable', 'Аутентификация временно перегружена') + try: + digest = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024) + finally: + KDF_SLOTS.release() + return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex() + + +class Store: + persistent = True + + def __init__(self): + self.ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600')) + if not 1 <= self.ttl <= 86400: + raise ValueError('SESSION_TTL_SECONDS must be 1..86400') + self.access_mode = os.environ.get('DB_ACCESS_MODE', 'optimized') + if self.access_mode not in ('baseline', 'optimized'): + raise ValueError('DB_ACCESS_MODE must be baseline or optimized') + pool_size = int(os.environ.get('DB_POOL_MAX', '8')) + if not 1 <= pool_size <= 32: + raise ValueError('DB_POOL_MAX must be 1..32') + self.pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=pool_size, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=2000 -c lock_timeout=1500'}, open=True) + self.pool.wait(timeout=10) + tele.pool(self.pool) + self.cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.15, socket_timeout=0.15, decode_responses=True) + self.objects = Objects(setting) + self.dummy_hash = password_hash('dummy-password-for-unknown-users') + with self.connection() as conn: + if not conn.execute("SELECT version FROM schema_migrations WHERE 1=1 AND version = '006_observability.sql'").fetchone(): + raise RuntimeError('Run the separate migration service first') + + @contextlib.contextmanager + def connection(self): + try: + with tele.span('db.transaction', {'db.system':'postgresql'}, tele.SpanKind.CLIENT): + with self.pool.connection() as conn: + yield conn + except psycopg.errors.UniqueViolation: + raise APIError(409, 'conflict', 'Имя пользователя занято') from None + except (psycopg.Error, PoolTimeout): + raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None + + def close(self): + self.pool.close() + self.cache.close() + self.objects.client.close() + + def ready(self): + self.objects.ready() + with self.connection() as conn: + conn.execute('SELECT 1') + + def register(self, body): + fields(body, ('username', 'display_name', 'password')) + name, display = username(body['username']), text(body['display_name'], 100) + # Passwords may contain spaces and colons; only length/UTF-8 are restricted. + password = body['password'] + if not isinstance(password, str) or not 12 <= len(password) <= 128: + invalid() + try: + password.encode('utf-8') + except UnicodeError: + invalid() + hashed = password_hash(password) + with self.connection() as conn: + row = conn.execute(''' + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + ''', (uuid.uuid4(), name, display.encode('utf-8'), hashed)).fetchone() + return public(row) + + def login(self, headers): + authorization = headers.get('Authorization', '') + try: + scheme, encoded = authorization.split(' ', 1) + if scheme.lower() != 'basic': + unauthorized() + name, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + except (ValueError, UnicodeError): + unauthorized() + if not re_username(name) or not 12 <= len(password) <= 128: + unauthorized() + with self.connection() as conn: + row = conn.execute('SELECT id, username, display_name, created_at, password_hash FROM users WHERE 1=1 AND username = %s', (name,)).fetchone() + hashed = row['password_hash'] if row else self.dummy_hash + verified = hmac.compare_digest(password_hash(password, bytes.fromhex(hashed.split('$')[4])), hashed) + if row is None or not verified: + unauthorized() + token = secrets.token_urlsafe(32) + with self.connection() as conn: + expiry = conn.execute(''' + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, clock_timestamp() + %s * INTERVAL '1 second') + RETURNING expires_at + ''', (hashlib.sha256(token.encode()).hexdigest(), row['id'], self.ttl)).fetchone()['expires_at'] + return {'token': token, 'token_type': 'Bearer', 'expires_at': public({'expires_at': expiry})['expires_at'], 'user': public(row)} + + def token_hash(self, headers): + parts = headers.get('Authorization', '').split(' ') + if len(parts) != 2 or parts[0].lower() != 'bearer' or len(parts[1]) != 43 or not parts[1].isascii(): + unauthorized() + return hashlib.sha256(parts[1].encode()).hexdigest() + + def authenticate(self, headers): + token_hash = self.token_hash(headers) + with self.connection() as conn: + session = conn.execute(''' + SELECT user_id + FROM sessions + WHERE 1=1 + AND token_hash = %s + AND expires_at > clock_timestamp() + ''', (token_hash,)).fetchone() + if session is None: + unauthorized() + key = 'user:' + str(session['user_id']) + try: + cached = self.cache.get(key) + if cached: + return json.loads(cached) + except (redis.RedisError, ValueError): + pass + row = conn.execute('SELECT id, username, display_name, created_at FROM users WHERE 1=1 AND id = %s', (session['user_id'],)).fetchone() + if row is None: + unauthorized() + user = public(row) + try: + self.cache.setex(key, 60, json.dumps(user)) + except redis.RedisError: + pass + return user + + def logout(self, headers): + with self.connection() as conn: + conn.execute('DELETE FROM sessions WHERE 1=1 AND token_hash = %s', (self.token_hash(headers),)) + + def chat_in(self, conn, actor, chat_id): + row = conn.execute(''' + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE 1=1 + AND c.id = %s + AND cm.user_id = %s + ''', (chat_id, actor['id'])).fetchone() + if row is None: + missing() + members = conn.execute('SELECT user_id FROM chat_members WHERE 1=1 AND chat_id = %s ORDER BY user_id', (chat_id,)).fetchall() + return dict(public(row), member_ids=[str(v['user_id']) for v in members]) + + def chat(self, actor, chat_id): + with self.connection() as conn: + return self.chat_in(conn, actor, chat_id) + + def create_chat(self, actor, body): + fields(body, ('title', 'member_ids')) + title = text(body['title'], 100) + members = body['member_ids'] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + invalid() + members = [identifier(v) for v in members] + if len(set(members)) != len(members) or actor['id'] not in members: + invalid() + with self.connection() as conn: + chat_id = uuid.uuid4() + conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat_id, title.encode('utf-8'))) + # Checking after INSERT exercises rollback of the whole composite operation. + found = conn.execute('SELECT id FROM users WHERE 1=1 AND id = ANY(%s::uuid[])', (members,)).fetchall() + if len(found) != len(members): + invalid() + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', [(chat_id, v) for v in members]) + return self.chat_in(conn, actor, chat_id) + + def list_chats(self, actor, limit, cursor, scope): + with self.connection() as conn: + where = 'AND (c.created_at, c.id) > (%s::timestamptz, %s::uuid)' if cursor else '' + params = [actor['id']] + list(cursor or ()) + [limit + 1] + rows = conn.execute(''' + SELECT c.id + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE 1=1 + AND cm.user_id = %s + ''' + where + ''' + ORDER BY c.created_at, c.id + LIMIT %s + ''', params).fetchall() + return page([self.chat_in(conn, actor, v['id']) for v in rows], limit, scope) + + def require_member(self, conn, actor, chat_id): + if self.access_mode == 'baseline': + return self.chat_in(conn, actor, chat_id) + if not conn.execute(''' + SELECT 1 + FROM chat_members + WHERE 1=1 + AND chat_id = %s + AND user_id = %s + ''', (chat_id, actor['id'])).fetchone(): + missing() + + def send(self, actor, chat_id, body, key): + if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + invalid() + if not isinstance(body, dict) or set(body) - {'text', 'attachment_ids'} or 'text' not in body: + invalid() + attachments = body.get('attachment_ids', []) + if not isinstance(attachments, list) or len(attachments) > 10: + invalid() + attachments = [identifier(v) for v in attachments] + if len(set(attachments)) != len(attachments): + invalid() + content = body['text'] + if not isinstance(content, str) or len(content) > 4000 or (not attachments and not content.strip()): + invalid() + try: + content.encode('utf-8') + except UnicodeError: + invalid() + canonical = json.dumps({'text': content, 'attachment_ids': attachments}, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + digest = hashlib.sha256(canonical.encode('utf-8')).hexdigest() + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + # The unique key arbitrates concurrent processes; everything commits together. + claimed = conn.execute(''' + INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, expires_at) + VALUES (%s, %s, %s, %s, clock_timestamp() + INTERVAL '24 hours') + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET request_hash = EXCLUDED.request_hash + , message_id = NULL + , expires_at = EXCLUDED.expires_at + WHERE 1=1 + AND idempotency_keys.expires_at <= clock_timestamp() + RETURNING request_hash + ''', (actor['id'], chat_id, key, digest)).fetchone() + if claimed is None: + previous = conn.execute(''' + SELECT ik.request_hash + , m.id + , m.chat_id + , m.sender_id + , m.text + , m.created_at + , m.attachment_ids + FROM idempotency_keys ik + JOIN messages m ON m.id = ik.message_id + WHERE 1=1 + AND ik.user_id = %s + AND ik.chat_id = %s + AND ik.key = %s + ''', (actor['id'], chat_id, key)).fetchone() + if previous['request_hash'] != digest: + raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения') + previous.pop('request_hash') + return public(previous) + if attachments: + rows = conn.execute(''' + SELECT id, state + FROM attachments + WHERE 1=1 + AND id = ANY(%s::uuid[]) + AND chat_id = %s + ORDER BY id + FOR SHARE + ''', (attachments, chat_id)).fetchall() + if len(rows) != len(attachments): + missing() + if any(v['state'] not in ('ready', 'queued', 'processing') for v in rows): + raise APIError(409, 'conflict', 'Вложение ещё не готово') + # Per-chat row lock preserves committed history order across replicas/clocks. + stamp = conn.execute(''' + UPDATE chats + SET last_message_at = GREATEST(clock_timestamp(), last_message_at + INTERVAL '1 microsecond') + WHERE 1=1 + AND id = %s + RETURNING last_message_at + ''', (chat_id,)).fetchone()['last_message_at'] + row = conn.execute(''' + INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) + VALUES (%s, %s, %s, %s, %s, %s::uuid[]) + RETURNING id, chat_id, sender_id, text, created_at, attachment_ids + ''', (uuid.uuid4(), chat_id, actor['id'], content.encode('utf-8'), stamp, attachments)).fetchone() + if attachments: + with conn.cursor() as cur: + cur.executemany('INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)', [(row['id'], v) for v in attachments]) + conn.execute(''' + UPDATE idempotency_keys + SET message_id = %s + WHERE 1=1 + AND user_id = %s + AND chat_id = %s + AND key = %s + ''', (row['id'], actor['id'], chat_id, key)) + return public(row) + + def history(self, actor, chat_id, limit, cursor, scope): + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + where = 'AND (created_at, id) > (%s::timestamptz, %s::uuid)' if cursor else '' + params = [chat_id] + list(cursor or ()) + [limit + 1] + rows = conn.execute(''' + SELECT id, chat_id, sender_id, text, created_at, attachment_ids + FROM messages + WHERE 1=1 + AND chat_id = %s + ''' + where + ''' + ORDER BY created_at, id + LIMIT %s + ''', params).fetchall() + return page([public(v) for v in rows], limit, scope) + + def upload(self, actor, chat_id, name, source, length, mime, digest): + attachment_id, job_id = uuid.uuid4(), uuid.uuid4() + key = 'staging/' + str(attachment_id) + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + conn.execute('SELECT pg_advisory_xact_lock(2301005)') + pending = conn.execute("SELECT count(*) AS n FROM attachments WHERE 1=1 AND state IN ('uploading', 'queued', 'processing')").fetchone()['n'] + if pending >= int(os.environ.get('MAX_PENDING_JOBS', '100')): + raise APIError(503, 'unavailable', 'Очередь принятых работ заполнена') + conn.execute(''' + INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256, state) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading') + ''', (attachment_id, chat_id, actor['id'], name, 'results/' + str(attachment_id) + '/v1/original', key, mime, length, digest)) + with self.connection() as conn: + if conn.execute('SELECT id FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() is None: + unavailable() + self.objects.put(key, source, length, mime, digest) + conn.execute(''' + INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts, trace_context) + VALUES (%s, %s, 1, %s, %s, %s) + ''', (job_id, attachment_id, key, int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), Jsonb(tele.carrier()))) + row = conn.execute(''' + UPDATE attachments + SET state = 'queued' + , current_job_id = %s + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + RETURNING * + ''', (job_id, attachment_id)).fetchone() + tele.log('attachment_accepted', attachment_id=str(attachment_id), job_id=str(job_id)) + # Capture the queued snapshot, even if the worker finishes before HTTP reply. + return self.attachment_public(row) + + def attachment_public(self, row): + keys = ('id', 'chat_id', 'owner_id', 'filename', 'content_type', 'size_bytes', 'sha256', 'created_at', 'updated_at') + result = public({k: row[k] for k in keys}) + variants = [{k: v for k, v in item.items() if k != 'object_key'} for item in row['variants']] + return dict(result, status=row['state'], variants=variants, error_code=row['error_code']) + + def attachment(self, actor, attachment_id): + with self.connection() as conn: + row = conn.execute(''' + SELECT a.* + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id + WHERE 1=1 + AND a.id = %s + AND cm.user_id = %s + ''', (attachment_id, actor['id'])).fetchone() + if row is None: + missing() + if row['state'] == 'storage_missing': + unavailable() + if row['state'] == 'uploading': + raise APIError(409, 'conflict', 'Приём не завершён') + return row + + +def re_username(value): + import re + return re.fullmatch(r'[a-z0-9_]{3,32}', value) diff --git a/src/render.py b/src/render.py new file mode 100644 index 0000000..a909334 --- /dev/null +++ b/src/render.py @@ -0,0 +1,64 @@ +"""Isolated image decoder/resizer; no network, database or credentials.""" +import hashlib +import json +import math +from pathlib import Path +import resource +import sys +import warnings + +resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024)) +resource.setrlimit(resource.RLIMIT_CPU, (15, 16)) +from PIL import Image, ImageOps, PngImagePlugin +Image.MAX_IMAGE_PIXELS = 40_000_000 +PngImagePlugin.MAX_TEXT_CHUNK = PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024 +warnings.filterwarnings('error', category=Image.DecompressionBombWarning) + + +def render(source, directory, version): + output = [] + with Image.open(source, formats=['PNG', 'JPEG']) as image: + if image.width * image.height > 40_000_000: + raise ValueError('pixels') + image.load() + ImageOps.exif_transpose(image, in_place=True) + rgba = None + if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info): + rgba = image if image.mode == 'RGBA' else image.convert('RGBA') + rgb = Image.new('RGB', rgba.size, 'white') + alpha = rgba.getchannel('A') + rgb.paste(rgba, mask=alpha) + alpha.close() + else: + rgb = image if image.mode == 'RGB' else image.convert('RGB') + try: + for name, bound, fmt, quality in [('thumbnail', 256, 'JPEG', 85), ('crop', 128, 'JPEG', 85), ('optimized', 1280, 'WEBP', 82 if version == 1 else 70)]: + if name == 'crop': + result = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(.5, .5)) + else: + scale = min(1, bound / max(rgb.size)) + size = tuple(max(1, math.floor(v * scale)) for v in rgb.size) + result = rgb.resize(size, Image.Resampling.LANCZOS) + try: + path = directory / name + options = {'quality': quality, 'exif': b'', 'icc_profile': None} + if fmt == 'WEBP': + options['method'] = 4 + result.save(path, format=fmt, **options) + raw = path.read_bytes() + output.append({'name': name, 'width': result.width, 'height': result.height, 'content_type': 'image/jpeg' if fmt == 'JPEG' else 'image/webp', 'size_bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()}) + finally: + result.close() + finally: + if rgb is not image: + rgb.close() + if rgba is not None and rgba is not image: + rgba.close() + return output + + +if __name__ == '__main__': + try: + print(json.dumps(render(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3])))) + except (OSError, ValueError, SyntaxError, Image.DecompressionBombError, Image.DecompressionBombWarning): + raise SystemExit(2) diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..84049a9 --- /dev/null +++ b/src/server.py @@ -0,0 +1,317 @@ +"""Bounded threaded API; persistent correctness state is shared across replicas.""" +import concurrent.futures +import contextlib +import http.server +import json +import logging +import os +import time +import signal +import socket +import threading +import uuid +from urllib.parse import parse_qs, urlsplit + +from objects import bounded_upload, receive, filename, unavailable + +import telemetry as tele + +from common import APIError, identifier, invalid, missing, page_args + +LOG = logging.getLogger('messenger') + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + invalid() + result[key] = value + return result + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def setup(self): + super().setup() + self.connection.settimeout(3) + + def send_error(self, code, message=None, explain=None): + # BaseHTTPRequestHandler otherwise emits HTML without contract headers. + self.request_id = uuid.uuid4().hex + self.route = '' + self.close_connection = True + status = 413 if code == 413 else 400 + self.reply(status, {'error': {'code': 'payload_too_large' if status == 413 else 'invalid_request', 'message': 'Некорректный HTTP-запрос', 'request_id': self.request_id}}) + + def log_message(self, *args): + pass + + def reply(self, status, body): + if status == 200 and isinstance(body, tuple): + return self.download(*body) + raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8') + if status >= 400: + self.close_connection = True + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(raw))) + if self.close_connection: + self.send_header('Connection', 'close') + self.send_header('X-Request-Id', self.request_id) + self.send_header('X-Instance-Id', self.server.instance_id) + if status == 401 and self.server.store.persistent: + self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"') + self.end_headers() + if self.command != 'HEAD': + self.wfile.write(raw) + + def download(self, row, response, chunk): + stream = response['Body'] + try: + self.send_response(200) + self.send_header('Content-Type', row['content_type']) + self.send_header('Content-Length', str(row['size_bytes'])) + self.send_header('X-Request-Id', self.request_id) + self.send_header('X-Instance-Id', self.server.instance_id) + self.send_header('X-Content-Type-Options', 'nosniff') + self.send_header('Cache-Control', 'private, no-store') + self.end_headers() + deadline = time.monotonic() + 15 + while chunk: + self.wfile.write(chunk) + if time.monotonic() > deadline: + raise TimeoutError() + chunk = stream.read(64 * 1024) + finally: + stream.close() + + def body(self): + lengths = self.headers.get_all('Content-Length', []) + if len(lengths) != 1 or self.headers.get('Transfer-Encoding'): + invalid() + try: + length = int(lengths[0]) + except ValueError: + invalid() + if length > 32768: + raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт') + if length <= 0: + invalid() + if self.headers.get_content_type() != 'application/json': + raise APIError(415, 'unsupported_media_type', 'Требуется application/json') + try: + raw = self.rfile.read(length) + if len(raw) != length: + invalid() + return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid()) + except (ValueError, UnicodeError, RecursionError): + invalid() + + def dispatch(self): + self.request_id = uuid.uuid4().hex + self.route = urlsplit(self.path).path + template = tele.route(self.route) + method = self.command if self.command in ('GET','POST','DELETE','PUT','PATCH','HEAD','OPTIONS') else 'OTHER' + attrs = {'method':method, 'route':template} + started = time.monotonic() + context = tele.span(method+' '+template, {'http.request.method':method, 'http.route':template}, + tele.SpanKind.SERVER, {k:self.headers[k] for k in ('traceparent','tracestate') if k in self.headers}) + # Health probes do not enter business telemetry or create noisy dependency spans. + if template.startswith('/health/'): + context = tele.suppressed() + status = 500 + with context as current: + try: + status, result = self.route_request() + except APIError as exc: + status = exc.status + result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}} + except Exception: + result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}} + try: + self.reply(status, result) + except (BrokenPipeError, ConnectionResetError, TimeoutError, OSError): + self.close_connection = True + if not template.startswith('/health/'): + attrs['status_class'] = str(status//100)+'xx' + tele.count('http_server_requests', attrs) + tele.duration('http_server_duration', time.monotonic()-started, attrs) + if current is not None: + current.set_attribute('http.response.status_code', status) + if status >= 500: + current.set_status(tele.Status(tele.StatusCode.ERROR)) + tele.log('http_request', request_id=self.request_id, method=method, route=template, status=status, duration_seconds=round(time.monotonic()-started,6)) + + def route_request(self): + method, path, store = self.command, self.route, self.server.store + if method == 'GET' and path in ('/health/live', '/health/ready'): + if path.endswith('ready'): + store.ready() + return 200, {'status': 'ok'} + if method == 'POST' and path == '/api/v1/users': + return 201, store.register(self.body()) + if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions': + if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0': + invalid() + return 201, store.login(self.headers) + actor = store.authenticate(self.headers) + if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current': + store.logout(self.headers) + return 204, None + if method == 'GET' and path == '/api/v1/users/me': + return 200, actor + query = parse_qs(urlsplit(self.path).query, keep_blank_values=True) + if path == '/api/v1/chats': + if method == 'POST': + return 201, store.create_chat(actor, self.body()) + if method == 'GET': + scope = 'chats:' + actor['id'] + limit, cursor = page_args(query, scope) + return 200, store.list_chats(actor, limit, cursor, scope) + parts = path.split('/') + if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'attachments'] and method == 'GET': + row = store.attachment(actor, identifier(parts[4])) + if len(parts) == 5: + if query: + invalid() + return 200, store.attachment_public(row) + if parts[5] == 'content': + if set(query) - {'variant'} or any(len(v) != 1 for v in query.values()): + invalid() + if row['state'] != 'ready': + raise APIError(409, 'conflict', 'Вложение ещё не готово') + variant = query.get('variant', ['original'])[0] + if variant != 'original': + item = next((v for v in row['variants'] if v['name'] == variant), None) + if item is None: + missing() + row = dict(row, object_key=item['object_key'], content_type=item['content_type'], size_bytes=item['size_bytes']) + response = store.objects.get(row['object_key']) + try: + chunk = response['Body'].read(64 * 1024) + if response['ContentLength'] != row['size_bytes'] or not chunk: + unavailable() + except Exception: + response['Body'].close() + unavailable() + return 200, (row, response, chunk) + if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']: + chat_id = identifier(parts[4]) + if store.access_mode == 'baseline': + store.chat(actor, chat_id) + if len(parts) == 5 and method == 'GET': + return 200, store.chat(actor, chat_id) + if len(parts) == 6 and parts[5] == 'attachments' and method == 'POST': + store.chat(actor, chat_id) + if set(query) != {'filename'} or len(query['filename']) != 1: + invalid() + name = filename(query['filename'][0]) + with bounded_upload(): + with receive(self) as (source, length, mime, digest): + return 202, store.upload(actor, chat_id, name, source, length, mime, digest) + if len(parts) == 6 and parts[5] == 'messages': + if method == 'POST': + keys = self.headers.get_all('Idempotency-Key', []) + return 201, store.send(actor, chat_id, self.body(), keys[0] if len(keys) == 1 else None) + if method == 'GET': + scope = 'messages:' + chat_id + limit, cursor = page_args(query, scope) + return 200, store.history(actor, chat_id, limit, cursor, scope) + missing() + + do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch + + +class Server(http.server.HTTPServer): + request_queue_size = 64 + allow_reuse_address = True + + def __init__(self, address, store): + self.store = store + self.instance_id = tele.INSTANCE + workers = int(os.environ.get('HTTP_WORKERS', '16')) + if not 1 <= workers <= 128: + raise ValueError('HTTP_WORKERS must be 1..128') + self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers) + self.slots = threading.BoundedSemaphore(workers * 2) + self.connections = set() + self.connections_lock = threading.Lock() + super().__init__(address, Handler) + + def process_request(self, request, address): + if not self.slots.acquire(blocking=False): + rid = uuid.uuid4().hex + raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode() + try: + request.settimeout(0.2) + request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Instance-Id: ' + self.instance_id + '\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw) + except OSError: + pass + self.shutdown_request(request) + return + with self.connections_lock: + self.connections.add(request) + self.executor.submit(self.worker, request, address) + + def worker(self, request, address): + try: + self.finish_request(request, address) + except OSError: + pass + finally: + self.shutdown_request(request) + with self.connections_lock: + self.connections.discard(request) + self.slots.release() + + def close_connections(self): + with self.connections_lock: + connections = list(self.connections) + for connection in connections: + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + def server_close(self): + super().server_close() + # Bound draining even when a slow client keeps an incomplete request alive. + deadline = threading.Timer(5, self.close_connections) + deadline.start() + try: + self.executor.shutdown(wait=True) + finally: + deadline.cancel() + + +def main(): + logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s') + tele.configure('messenger-api') + from postgres import Store + store = Store() + server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store) + stopping = threading.Event() + + def stop(signum, frame): + if not stopping.is_set(): + stopping.set() + LOG.info('shutdown_started signal=%s', signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGUSR1, lambda *_: LOG.warning('pool_stats instance=%s stats=%s', server.instance_id, json.dumps(store.pool.get_stats()))) + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + LOG.info('started port=%s', server.server_address[1]) + try: + server.serve_forever(poll_interval=0.1) + finally: + server.server_close() + store.close() + tele.shutdown() + LOG.info('shutdown_complete') + + +if __name__ == '__main__': + main() diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..ccc5b3a --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,217 @@ +"""Allowlisted manual SDK telemetry; bounded exporters never carry business payloads.""" +import contextlib +import contextvars +import datetime as dt +import json +import logging +import os +from pathlib import Path +import re +import resource +import threading +import time +import uuid + +from opentelemetry import trace, metrics +from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider, SpanProcessor +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult +from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + +INSTANCE = uuid.uuid4().hex +ACTIVE = False +SUPPRESSED = contextvars.ContextVar('telemetry_suppressed', default=False) +PROVIDERS = [] +COUNTERS = {} +HISTOGRAMS = {} +STATE = {'worker_busy': 0, 'queue_depth': 0, 'queue_oldest_age': 0, 'queue_observation_ok': 1, + 'spans_finished': 0, 'spans_exported': 0, 'spans_export_failed': 0} +LOCK = threading.Lock() +PROPAGATOR = TraceContextTextMapPropagator() +LOGGER = logging.getLogger('messenger.telemetry') +TRACER = trace.get_tracer('messenger') +POOL = None +SERVICE = '' +BUCKETS = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 60] + +class FinishedSpans(SpanProcessor): + def on_end(self, span): + with LOCK: + STATE['spans_finished'] += 1 + +class CountedExporter(SpanExporter): + def __init__(self, delegate): + self.delegate = delegate + + def export(self, spans): + result = self.delegate.export(spans) + with LOCK: + STATE['spans_exported' if result == SpanExportResult.SUCCESS else 'spans_export_failed'] += len(spans) + return result + + def shutdown(self): + self.delegate.shutdown() + + def force_flush(self, timeout_millis=30000): + return self.delegate.force_flush(timeout_millis) + +def configure(service): + global ACTIVE, TRACER, SERVICE + if PROVIDERS or os.environ.get('TELEMETRY_ENABLED', '1') == '0': + return + SERVICE = service + ratio = float(os.environ.get('TRACE_SAMPLE_RATIO', '0.25')) + if not 0 <= ratio <= 1: + raise ValueError('TRACE_SAMPLE_RATIO must be 0..1') + resource_attributes = Resource.create({ + 'service.name': service, 'service.version': 'lab6.1', + 'service.instance.id': INSTANCE, 'deployment.environment.name': 'lab', + }) + endpoint = os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/') + provider = TracerProvider(resource=resource_attributes, sampler=ParentBased(TraceIdRatioBased(ratio))) + provider.add_span_processor(FinishedSpans()) + provider.add_span_processor(BatchSpanProcessor( + CountedExporter(OTLPSpanExporter(endpoint=endpoint+'/v1/traces', timeout=1)), + max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=500, + export_timeout_millis=1500)) + trace.set_tracer_provider(provider) + TRACER = trace.get_tracer('messenger') + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=endpoint+'/v1/metrics', timeout=1), + export_interval_millis=2000, export_timeout_millis=1500) + meter_provider = MeterProvider(resource=resource_attributes, metric_readers=[reader], + views=[View(instrument_name='*duration', aggregation=ExplicitBucketHistogramAggregation(BUCKETS))]) + metrics.set_meter_provider(meter_provider) + meter = metrics.get_meter('messenger') + for name in ['http_server_requests', 'jobs_processed', 'job_retries']: + COUNTERS[name] = meter.create_counter(name) + for name in ['http_server_duration', 'job_processing_duration', 'job_ready_duration']: + HISTOGRAMS[name] = meter.create_histogram(name, unit='s') + def gauge(name, getter, unit=''): + def observe(options): + try: + yield metrics.Observation(getter(), {}) + except (OSError, ValueError): + return + meter.create_observable_gauge(name, callbacks=[observe], unit=unit) + for name in ['queue_depth', 'queue_oldest_age', 'queue_observation_ok', 'worker_busy', + 'spans_finished', 'spans_exported', 'spans_export_failed']: + gauge(name, lambda name=name: STATE[name], 's' if name=='queue_oldest_age' else '') + gauge('worker_capacity', lambda: int(os.environ.get('WORKER_CONCURRENCY', '1')) if service=='messenger-worker' else 0) + gauge('db_pool_used', lambda: max(0, POOL.get_stats().get('pool_size',0)-POOL.get_stats().get('pool_available',0)) if POOL else 0) + gauge('db_pool_waiting', lambda: POOL.get_stats().get('requests_waiting',0) if POOL else 0) + gauge('db_pool_capacity', lambda: int(os.environ.get('DB_POOL_MAX','8')) if POOL else 0) + gauge('process_resident_memory', lambda: int(Path('/proc/self/statm').read_text().split()[1])*os.sysconf('SC_PAGE_SIZE'), 'By') + gauge('container_memory', lambda: int(Path('/sys/fs/cgroup/memory.current').read_text()), 'By') + def cpu(options): + use = resource.getrusage(resource.RUSAGE_SELF) + yield metrics.Observation(use.ru_utime+use.ru_stime, {}) + meter.create_observable_counter('process_cpu', callbacks=[cpu], unit='s') + log_provider = LoggerProvider(resource=resource_attributes) + log_provider.add_log_record_processor(BatchLogRecordProcessor( + OTLPLogExporter(endpoint=endpoint+'/v1/logs', timeout=1), + max_queue_size=256, max_export_batch_size=64, schedule_delay_millis=500, + export_timeout_millis=1500)) + LOGGER.setLevel(logging.INFO) + LOGGER.propagate = False + LOGGER.addHandler(LoggingHandler(level=logging.INFO, logger_provider=log_provider)) + PROVIDERS.extend([provider, meter_provider, log_provider]) + ACTIVE = True + +def pool(value): + global POOL + POOL = value + +def route(path): + if path in ['/health/live', '/health/ready', '/api/v1/users', '/api/v1/users/me', + '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats']: + return path + patterns = [ + (r'/api/v1/chats/[^/]+/messages', '/api/v1/chats/{chat_id}/messages'), + (r'/api/v1/chats/[^/]+/attachments', '/api/v1/chats/{chat_id}/attachments'), + (r'/api/v1/chats/[^/]+', '/api/v1/chats/{chat_id}'), + (r'/api/v1/attachments/[^/]+/content', '/api/v1/attachments/{attachment_id}/content'), + (r'/api/v1/attachments/[^/]+', '/api/v1/attachments/{attachment_id}'), + ] + return next((template for pattern,template in patterns if re.fullmatch(pattern,path)), 'unmatched') + +@contextlib.contextmanager +def span(name, attributes=None, kind=SpanKind.INTERNAL, carrier=None): + if not ACTIVE or SUPPRESSED.get() or (name == 'db.transaction' and not trace.get_current_span().get_span_context().is_valid): + yield trace.get_current_span() + return + context = PROPAGATOR.extract(carrier) if carrier is not None else None + with TRACER.start_as_current_span(name, attributes=attributes or {}, kind=kind, + context=context, record_exception=False, set_status_on_exception=False) as current: + try: + yield current + except Exception as error: + current.set_attribute('error.type', type(error).__name__) + current.set_status(Status(StatusCode.ERROR)) + log('operation_failed', error_type=type(error).__name__) + raise + +@contextlib.contextmanager +def suppressed(): + token = SUPPRESSED.set(True) + try: + yield None + finally: + SUPPRESSED.reset(token) + +def carrier(): + result = {} + if ACTIVE: + PROPAGATOR.inject(result) + return result + +def log(event, **attributes): + if not ACTIVE: + return + # Callers pass only fixed event names, bounded outcomes and technical correlation IDs. + context = trace.get_current_span().get_span_context() + body = {'timestamp': dt.datetime.now(dt.timezone.utc).isoformat(), + 'severity': 'INFO', 'service.name': SERVICE, 'event': event, + 'sampled': bool(context.trace_flags.sampled), 'trace_id': format(context.trace_id,'032x'), 'span_id': format(context.span_id,'016x'), + **attributes} + LOGGER.info(json.dumps(body, ensure_ascii=True, separators=(',',':'))) + +def count(name, attributes=None): + if ACTIVE: + COUNTERS[name].add(1, attributes or {}) + +def duration(name, seconds, attributes=None): + if ACTIVE: + HISTOGRAMS[name].record(max(0, seconds), attributes or {}) + +def flush(): + for provider in PROVIDERS: + provider.force_flush(timeout_millis=5000) + +def shutdown(): + for provider in PROVIDERS: + provider.shutdown() + +class S3Client: + """Span each SDK request without serializing args, keys, URLs or SDK exceptions.""" + def __init__(self, client): + self.client = client + + def __getattr__(self, name): + method = getattr(self.client,name) + if name not in {'head_object','get_object','put_object','delete_object','head_bucket','create_bucket','list_objects_v2'}: + return method + def call(*args, **kwargs): + with span('s3.'+name, {'rpc.system':'aws-api','rpc.service':'S3','rpc.method':name}, SpanKind.CLIENT): + return method(*args, **kwargs) + return call diff --git a/student-tests/durability.py b/student-tests/durability.py new file mode 100644 index 0000000..3470de0 --- /dev/null +++ b/student-tests/durability.py @@ -0,0 +1,52 @@ +"""Stop worker, accept queued, restart API/broker, then complete the same attachment.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client,Failure + + +def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip() + + +def main(): + c=Client();alice,bob,eve,chat=c.bootstrap();record={} + before=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True)) + try: + docker('compose','stop','worker','dispatcher') + queued,raw=c.attachment(alice,chat) + assert queued['status']=='queued' + message=c.message(alice,chat,'accepted with worker stopped',attachments=[queued['id']])['json'] + c.call('GET','/api/v1/attachments/'+queued['id']+'/content',409,actor=bob) + killed=docker('compose','ps','-q','api').splitlines() + for container in killed:docker('kill','--signal=KILL',container) + record['killed_api_containers']=killed + docker('compose','up','-d','--wait','--no-deps','api') + time.sleep(2) + assert c.call('GET','/api/v1/attachments/'+queued['id'],actor=bob,schema='Attachment')['json']['status']=='queued' + docker('compose','up','-d','--no-deps','dispatcher') + deadline=time.monotonic()+10 + while int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))<1: + assert time.monotonic()=length_before>=1 + record['stream_entries_before_after_broker_restart']=[length_before,length_after] + docker('compose','up','-d','--no-deps','worker','dispatcher') + ready=c.wait_ready(bob,queued) + assert c.call('GET','/api/v1/attachments/'+queued['id']+'/content',actor=bob)['bytes']==raw + record.update(attachment_id=queued['id'],original_sha256=hashlib.sha256(raw).hexdigest(),status=ready['status'],variants=[v['name'] for v in ready['variants']],old_bearer_survived=True,message_attachment_retained=message['attachment_ids']==[queued['id']]) + finally:docker('compose','up','-d','--no-deps','worker','dispatcher') + after=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True)) + record['before']=before;record['after']=after + print(json.dumps(record,indent=2)) + + +if __name__=='__main__':main() diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 0000000..906356f --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,24 @@ +"""Small deterministic PNG helpers; generation happens outside measurement.""" +import random +import struct +import zlib + + +def chunk(name, data): + return struct.pack('!I', len(data)) + name + data + struct.pack('!I', zlib.crc32(name + data) & 0xffffffff) + + +def png(width, height, noise=False, rgb=True): + channels = 3 if rgb else 1 + rand = random.Random(2301) + encoded = zlib.compressobj(1) + parts = [] + for _ in range(height): + line = bytes(rand.getrandbits(8) for _ in range(width * channels)) if noise else b'\x7f' * (width * channels) + parts.append(encoded.compress(b'\0' + line)) + parts.append(encoded.flush()) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2 if rgb else 0, 0, 0, 0)) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'') + + +def bomb(): + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', 10000, 4001, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'x')) + chunk(b'IEND', b'') diff --git a/student-tests/observability.py b/student-tests/observability.py new file mode 100644 index 0000000..5b8a624 --- /dev/null +++ b/student-tests/observability.py @@ -0,0 +1,257 @@ +"""Real OTLP evidence, controlled incidents and bounded telemetry-backend outage.""" +import argparse +import base64 +import concurrent.futures +import json +import os +from pathlib import Path +import subprocess +import sys +import time +from urllib.parse import urlencode +import urllib.request +import uuid + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure + +BASE=os.environ.get('BASE_URL','http://127.0.0.1:18093') +PROM=os.environ.get('PROMETHEUS_URL','http://127.0.0.1:19096') +LOKI=os.environ.get('LOKI_URL','http://127.0.0.1:13106') +TEMPO=os.environ.get('TEMPO_URL','http://127.0.0.1:13206') +COLLECTOR=os.environ.get('COLLECTOR_URL','http://127.0.0.1:18896') +OPENER=urllib.request.build_opener(urllib.request.ProxyHandler({})) + +def get(url): + req=urllib.request.Request(url,headers={'Accept':'application/json'}) + with OPENER.open(req,timeout=10) as response: + return json.load(response) + +def prom(query): + return get(PROM+'/api/v1/query?'+urlencode({'query':query}))['data']['result'] + +def scalar(query): + rows=prom(query) + return float(rows[0]['value'][1]) if rows else 0 + +def logs(query, start=None, limit=2000): + params={'query':query,'start':str(int((start or time.time()-600)*1e9)), + 'end':str(time.time_ns()),'limit':str(limit),'direction':'forward'} + result=get(LOKI+'/loki/api/v1/query_range?'+urlencode(params))['data']['result'] + return [json.loads(value) for stream in result for stamp,value in stream['values']] + +def trace(identifier): + return get(TEMPO+'/api/traces/'+identifier) + +def spans(data): + result=[] + for batch in data.get('batches',data.get('resourceSpans',[])): + service=next(a['value']['stringValue'] for a in batch['resource']['attributes'] if a['key']=='service.name') + for scope in batch.get('scopeSpans',[]): + for span in scope['spans']: + result.append(dict(span,service=service)) + return result + +def wait(function,timeout=60): + end=time.monotonic()+timeout + while time.monotonic()=4 else None) + assert {r['event'] for r in records}>={'http_request','attachment_accepted','job_published','job_processed'} + assert any(r.get('request_id')==response['headers'].get('X-Request-Id') for r in records) + assert all(len(r['trace_id'])==32 and len(r['span_id'])==16 for r in records) + all_logs=logs('{service_name=~"messenger-.*"}',start=proof_start,limit=5000) + payload=json.dumps(all_logs)+json.dumps(data) + for value in [marker,a['token'],a['registration']['password'],a['registration']['username'],'private-canary.png']: + assert value not in payload, 'business/credential canary leaked' + metric_rows=prom('{service_name=~"messenger-.*"}') + forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','job_id','trace_id','span_id'} + assert all(not forbidden.intersection(row['metric']) for row in metric_rows) + names={r['metric']['__name__'] for r in metric_rows} + assert {'http_server_duration_seconds_bucket','jobs_processed_total','queue_depth','queue_oldest_age_seconds','db_pool_used','worker_capacity'}<=names + save('trace-proof.json',{'trace_id':tid,'request_id':response['headers'].get('X-Request-Id'), + 'status':ready['status'],'consumer_parent_matches_producer':True,'producer_descends_from_upload':True, + 'api_worker_logs_correlated':True,'secret_canaries_absent':True,'metric_labels_allowlisted':True, + 'trace':data,'logs':records}) + # Retry is a real S3 outage after durable acceptance, not fabricated spans. + compose('stop','worker') + retry_tid=uuid.uuid4().hex + queued=upload(c,a,chat,retry_tid)['json'] + paused=False + try: + compose('pause','s3');paused=True + compose('up','-d','--no-deps','worker') + def attempted(): + raw=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc', + "SELECT attempt FROM attachment_jobs WHERE 1=1 AND attachment_id = '"+queued['id']+"';") + return int(raw)>=1 + wait(attempted) + time.sleep(5) + finally: + if paused:compose('unpause','s3') + compose('up','-d','--no-deps','worker') + wait(lambda:c.call('GET','/health/ready',schema='Health')) + c.wait_ready(a,queued) + def retried(): + data=trace(retry_tid) + return data if len([s for s in spans(data) if s['kind']=='SPAN_KIND_CONSUMER'])>=2 else None + retry_data=wait(retried) + retry_records=wait(lambda:logs('{service_name="messenger-worker"} | json | trace_id="'+retry_tid+'" | event="job_retry"')) + save('retry-trace.json',{'trace_id':retry_tid,'retry_observed':True,'trace':retry_data,'logs':retry_records}) + print('PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.') + +def alerts(): + return get(PROM+'/api/v1/alerts')['data']['alerts'] + +def firing(name): + return [a for a in alerts() if a['labels']['alertname']==name and a['state']=='firing'] + +def incidents(): + c=Client(BASE);a,b,e,chat=c.bootstrap();result={} + start=time.time() + compose('stop','worker') + try: + queued=[c.attachment(a,chat)[0] for _ in range(3)] + observed=wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')>=3) + fired=wait(lambda:firing('QueueLag')) + detected=time.time() + metrics={q:prom(q) for q in ['queue_depth','queue_oldest_age_seconds','worker_busy','worker_capacity']} + result['worker_stop']={'injected_at':start,'detected_at':detected, + 'detection_seconds':detected-start,'firing':fired,'metrics':metrics, + 'diagnosis':'Backlog and oldest age grow; worker completion stops while API acceptance still succeeds. Gauges expire after 30s.'} + finally: + compose('up','-d','--no-deps','worker') + for att in queued:c.wait_ready(a,att) + wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')==0) + wait(lambda:not firing('QueueLag')) + result['worker_stop']['resolved_at']=time.time() + # Keep readiness SELECT 1 available while real session SQL is blocked. + lock=subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1', + '-c',"BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(35); COMMIT;"], + stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + time.sleep(1) + db_start=time.time();stop=False;errors=[];error_trace=uuid.uuid4().hex + def traffic(): + while not stop: + r=c.call('GET','/api/v1/users/me',[200,503],actor=a, + headers={'traceparent':'00-'+error_trace+'-0123456789abcdef-01'}) + if r['status']==503:errors.append(r['elapsed']) + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + futures=[pool.submit(traffic) for _ in range(4)] + try: + fired=wait(lambda:firing('APIHighErrorRate'),timeout=50) + detected=time.time() + result['db_lock']={'injected_at':db_start,'detected_at':detected,'detection_seconds':detected-db_start, + 'firing':fired,'error_latencies_seconds':errors.copy(),'trace_id':error_trace} + lock.wait(timeout=45) + finally: + stop=True + for future in futures:future.result() + finally: + stop=True + if lock.poll() is None: + lock.terminate();lock.wait(timeout=5) + wait(lambda:c.call('GET','/api/v1/users/me',actor=a,schema='User')) + wait(lambda:not firing('APIHighErrorRate'),timeout=70) + data=wait(lambda:trace(error_trace)) + assert any(s['name']=='db.transaction' and s.get('status',{}).get('code')=='STATUS_CODE_ERROR' for s in spans(data)) + result['db_lock'].update(resolved_at=time.time(),trace=data, + diagnosis='Server 5xx spans contain slow/error PostgreSQL transaction; health remains available; sessions lock is the bottleneck.') + save('incidents.json',result) + print('PASS: worker-stop and real DB-lock incidents, local firing/resolved and causal DB traces.') + +def collector_metrics(): + with OPENER.open(COLLECTOR+'/metrics',timeout=5) as r:raw=r.read().decode() + result={} + for line in raw.splitlines(): + if line.startswith('otelcol_') and not line.startswith('#'): + name=line.split('{')[0].split(' ')[0] + try:result[name]=result.get(name,0)+float(line.rsplit(' ',1)[-1]) + except ValueError:pass + return result + +def backend_outage(): + c=Client(BASE);a,b,e,chat=c.bootstrap();before=collector_metrics() + cid=compose('ps','-q','otel-collector') + samples=[];messages=[];attachments=[];start=time.time() + compose('stop','tempo') + try: + for i in range(45): + for _ in range(3): + messages.append(c.message(a,chat,'synthetic-backend-outage')['json']['id']) + if i%10==0: + att,_=c.attachment(a,chat);attachments.append(c.wait_ready(a,att)['id']) + stat=json.loads(subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',cid],text=True)) + samples.append({'elapsed':time.time()-start,'memory':stat['MemUsage'],'cpu':stat['CPUPerc']}) + time.sleep(1) + during=collector_metrics() + assert during.get('otelcol_exporter_send_failed_spans_total',0)>before.get('otelcol_exporter_send_failed_spans_total',0) + assert during.get('otelcol_exporter_queue_size',0)<=128*2 + history=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100',actor=b,schema='MessagePage')['json'] + got={m['id'] for m in history['items']} + if history['next_cursor']: + history2=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+history['next_cursor'],actor=b,schema='MessagePage')['json'] + got.update(m['id'] for m in history2['items']) + assert set(messages)<=got + finally:compose('up','-d','--no-deps','tempo') + wait(lambda:get(TEMPO+'/api/search')) + tid=uuid.uuid4().hex + upload(c,a,chat,tid) + wait(lambda:trace(tid)) + save('backend-outage.json',{'duration_seconds':time.time()-start,'confirmed_messages':len(messages), + 'confirmed_attachments_ready':len(attachments),'all_confirmed_messages_retained':True, + 'collector_samples':samples,'before':before,'during':during,'recovered_trace_id':tid}) + print('PASS: Tempo unavailable, finite Collector buffers/drops, API writes/jobs retained, fresh trace after recovery.') + +if __name__=='__main__': + parser=argparse.ArgumentParser() + parser.add_argument('phase',choices=['proof','incidents','backend']) + args=parser.parse_args() + {'proof':proof,'incidents':incidents,'backend':backend_outage}[args.phase]() diff --git a/student-tests/observability_cost.py b/student-tests/observability_cost.py new file mode 100644 index 0000000..830402e --- /dev/null +++ b/student-tests/observability_cost.py @@ -0,0 +1,193 @@ +"""Fixed-arrival mixed workload: SDK off, full traces, 10% and 25% head sampling.""" +import concurrent.futures +import http.client +import json +import os +from pathlib import Path +import socket +import statistics +import subprocess +import sys +import threading +import time +from urllib.parse import urlsplit +import uuid + +sys.path.insert(0,str(Path(__file__).resolve().parent)) +from observability import Client, BASE, compose, collector_metrics, scalar, prom, logs, wait, save + +class DockerHTTP(http.client.HTTPConnection): + def connect(self): + self.sock=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) + self.sock.settimeout(8) + self.sock.connect(DOCKER_SOCKET) + +context=json.loads(subprocess.check_output(['docker','context','inspect'],text=True))[0] +DOCKER_SOCKET=urlsplit(context['Endpoints']['docker']['Host']).path + +def stats(container): + connection=DockerHTTP('localhost') + try: + connection.request('GET','/containers/'+container+'/stats?stream=false&one-shot=true') + response=connection.getresponse() + assert response.status==200 + data=json.load(response) + finally:connection.close() + memory=data.get('memory_stats',{}) + details=memory.get('stats',{}) + usage=memory.get('usage',0) + return {'cpu_seconds':data['cpu_stats']['cpu_usage']['total_usage']/1e9, + 'anonymous_bytes':details.get('anon',details.get('rss',0)), + 'working_set_bytes':max(0,usage-details.get('inactive_file',details.get('total_inactive_file',0))), + 'rx_bytes':sum(n.get('rx_bytes',0) for n in data.get('networks',{}).values())} + +def span_counts(): + return {r['metric']['service_instance_id']:float(r['value'][1]) + for r in prom('spans_finished{service_name=~"messenger-.*"}')} + +def delta(previous,current): + return sum(max(0,value-previous.get(key,0)) for key,value in current.items()) + +def quantile(values,q): + return sorted(values)[min(len(values)-1,int((len(values)-1)*q))] + +def set_mode(enabled,ratio): + env=os.environ.copy() + env.update(TELEMETRY_ENABLED=str(enabled),TRACE_SAMPLE_RATIO=str(ratio)) + subprocess.run(['docker','compose','up','-d','--wait','--no-deps','api','worker','dispatcher'],env=env, + check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + c=Client(BASE) + wait(lambda:c.call('GET','/health/ready',schema='Health')) + return c + +def probes(c,actor,mode): + results=[] + for duration in [3.5,3.5,3.5,1.2]: + query="BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep("+str(duration)+"); COMMIT;" + lock=subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',query], + stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + try: + def locked(): + value=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc', + "SELECT count(*) FROM pg_locks WHERE 1=1 AND relation = 'sessions'::regclass AND mode = 'AccessExclusiveLock' AND granted;") + return int(value)>0 + wait(locked,timeout=5) + result=c.call('GET','/api/v1/users/me',[200,503],actor=actor) + assert result['status']==(503 if duration>3 else 200) + assert result['elapsed']>.1 + results.append({'request_id':result['headers'].get('X-Request-Id'), + 'status':result['status'],'latency_seconds':result['elapsed']}) + finally:lock.wait(timeout=5) + if mode!='off': + time.sleep(3) + for row in results: + records=logs('{service_name="messenger-api"} | json | request_id="'+row['request_id']+'"',limit=10) + assert records + row['trace_sampled']=records[0]['sampled'] + row['trace_id']=records[0]['trace_id'] + else: + for row in results:row['trace_sampled']=False + return results + +def run(mode,enabled,ratio,repeat,seconds,rps): + c=set_mode(enabled,ratio);a,b,e,chat=c.bootstrap() + for _ in range(10):c.call('GET','/api/v1/users/me',actor=a) + time.sleep(5) + names=compose('ps','-q','api','worker','dispatcher','otel-collector','postgres','loki','tempo','prometheus','grafana','s3','s3-store','broker','lb','redis').splitlines() + labels={} + for cid in names: + label=json.loads(subprocess.check_output(['docker','inspect',cid,'--format','{{json .Config.Labels}}'],text=True)) + labels[cid]=label['com.docker.compose.service'] + before={cid:stats(cid) for cid in names} + ingest_before=collector_metrics();spans_before=span_counts() + memory={cid:[] for cid in names};sampling_stop=threading.Event() + def sample(): + while not sampling_stop.is_set(): + for cid in names: + try:memory[cid].append(stats(cid)) + except OSError:pass + sampling_stop.wait(1) + sampler=threading.Thread(target=sample,daemon=True);sampler.start() + latencies=[];failures=[];attachments=[];start=time.monotonic() + def request(i): + try: + if i%20==0: + result=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=cost.png',202, + actor=a,raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes(), + headers={'Content-Type':'image/png'},schema='Attachment') + return result['elapsed'],result['json'] + if i%5==0: + result=c.message(a,chat,'fixed-synthetic-cost-payload') + else:result=c.call('GET','/api/v1/users/me',actor=a,schema='User') + return result['elapsed'],None + except Exception as error:return None,type(error).__name__ + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + futures=[] + for i in range(seconds*rps): + delay=start+i/rps-time.monotonic() + if delay>0:time.sleep(delay) + futures.append(pool.submit(request,i)) + for future in futures: + elapsed,att=future.result() + if elapsed is None:failures.append(att) + else: + latencies.append(elapsed) + if att:attachments.append(att) + load_wall=time.monotonic()-start + for att in attachments:c.wait_ready(a,att) + # Drain exporter batches and metric-reader intervals, then compare counters by instance. + time.sleep(5) + sampling_stop.set();sampler.join(timeout=15) + after={cid:stats(cid) for cid in names} + ingest_after=collector_metrics();spans_after=span_counts() + assert not failures, failures + resource={} + for cid in names: + service=labels[cid] + target=resource.setdefault(service,{'cpu_seconds':0,'peak_sampled_anonymous_bytes':0,'peak_sampled_working_set_bytes':0,'rx_bytes':0}) + target['cpu_seconds']+=max(0,after[cid]['cpu_seconds']-before[cid]['cpu_seconds']) + target['rx_bytes']+=max(0,after[cid]['rx_bytes']-before[cid]['rx_bytes']) + target['peak_sampled_anonymous_bytes']+=max((s['anonymous_bytes'] for s in memory[cid]),default=0) + target['peak_sampled_working_set_bytes']+=max((s['working_set_bytes'] for s in memory[cid]),default=0) + counts={k:max(0,ingest_after.get(k,0)-ingest_before.get(k,0)) for k in [ + 'otelcol_receiver_accepted_spans_total','otelcol_receiver_accepted_log_records_total', + 'otelcol_receiver_accepted_metric_points_total','otelcol_exporter_sent_spans_total', + 'otelcol_exporter_send_failed_spans_total']} + finished=delta(spans_before,spans_after) + accepted=counts['otelcol_receiver_accepted_spans_total'] + result={'mode':mode,'enabled':enabled,'sample_ratio':ratio,'repeat':repeat, + 'arrival_rps':rps,'scheduled_requests':seconds*rps,'confirmed_requests':len(latencies), + 'load_wall_seconds':load_wall,'successful_rps':len(latencies)/load_wall, + 'p50_seconds':quantile(latencies,.5),'p95_seconds':quantile(latencies,.95),'p99_seconds':quantile(latencies,.99), + 'errors':failures,'image_jobs_ready':len(attachments),'resource':resource,'collector_ingest':counts, + 'sampled_spans_finished':finished,'sampled_spans_not_accepted_after_drain':max(0,finished-accepted)} + result['rare_probes']=probes(c,a,mode) + print(mode,repeat,'p95',round(result['p95_seconds'],5),'ingested spans',accepted,flush=True) + return result + +def main(): + import argparse + parser=argparse.ArgumentParser() + parser.add_argument('--seconds',type=int,default=20) + parser.add_argument('--rps',type=int,default=30) + parser.add_argument('--repeats',type=int,default=3) + args=parser.parse_args() + result={'method':'Fixed arrival mixed requests, 5% image uploads, 15% messages, 80% profile reads; same fixture/resources. No transparent retries.', + 'notes':['CPU cgroup total across load plus exporter drain; includes renderer.', + 'Anonymous/working-set memory sampled at 1Hz through read-only Docker stats API, not exact per-PID peak RSS.', + 'Collector rx bytes include OTLP HTTP, Prometheus scrapes and backend replies; accepted signal counts are exact Collector counter deltas.', + 'Fresh API/worker/dispatcher containers per setting; data/backends retained. Per-instance counter deltas ignore expired old series.', + 'Rare SQL-lock probes run after resource/ingest measurement; three SQL-lock probes and one slow success per run; future runs synchronize on pg_locks.'], + 'runs':[]} + modes=[('off',0,0),('full',1,1),('sample10',1,.1),('sample25',1,.25)] + try: + for repeat in range(args.repeats): + # Rotate order to reduce a fixed warm-cache/time-order advantage. + order=modes[repeat%len(modes):]+modes[:repeat%len(modes)] + for mode,enabled,ratio in order: + result['runs'].append(run(mode,enabled,ratio,repeat,args.seconds,args.rps)) + save('observability-cost.json',result) + finally:set_mode(1,.25) + print('PASS: telemetry cost experiment completed; balanced 25% tracing restored.') + +if __name__=='__main__':main() diff --git a/student-tests/persistence_telemetry.py b/student-tests/persistence_telemetry.py new file mode 100644 index 0000000..ffb48a1 --- /dev/null +++ b/student-tests/persistence_telemetry.py @@ -0,0 +1,37 @@ +"""Restart all stores and verify historical telemetry alongside public business persistence.""" +import json +import os +from pathlib import Path +import subprocess +import time +from urllib.parse import urlencode +from observability import BASE,PROM,trace,spans,logs,get,wait,save + +root=Path(__file__).resolve().parents[1] +identifier=json.loads((root/'evidence/trace-proof.json').read_text())['trace_id'] +before_trace=trace(identifier) +before_ids={s['spanId'] for s in spans(before_trace)} +query='{service_name=~"messenger-.*"} | json | trace_id="'+identifier+'"' +before_logs=logs(query,start=time.time()-3600) +assert before_logs +at=time.time()-5 +url=PROM+'/api/v1/query?'+urlencode({'query':'sum(http_server_requests_total)','time':str(at)}) +before_metric=get(url)['data']['result'] +assert before_metric +command=['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh','BASE_URL='+BASE] +result=subprocess.run(command,capture_output=True,text=True) +(root/'evidence/persistence.txt').write_text(result.stdout+result.stderr) +assert result.returncode==0,result.stdout[-300:] +after_trace=wait(lambda:trace(identifier),timeout=90) +after_ids={s['spanId'] for s in spans(after_trace)} +assert before_ids<=after_ids +after_logs=wait(lambda:logs(query,start=time.time()-3600),timeout=90) +assert {json.dumps(r,sort_keys=True) for r in before_logs}<={json.dumps(r,sort_keys=True) for r in after_logs} +after_metric=wait(lambda:get(url)['data']['result']) +assert before_metric==after_metric +save('telemetry-persistence.json',{'trace_id':identifier,'historical_span_count':len(before_ids), + 'historical_logs':len(before_logs),'fixed_metric_query_time':at, + 'metric_before':before_metric,'metric_after':after_metric, + 'business_data_and_existing_session_survived':True,'all_telemetry_volumes_restarted':True, + 'trace_log_metric_history_survived':True}) +print('PASS: full business restart, original Bearer, historical Tempo trace, Loki logs and fixed-time Prometheus sample.') diff --git a/student-tests/pipeline_checks.py b/student-tests/pipeline_checks.py new file mode 100644 index 0000000..c4d36ba --- /dev/null +++ b/student-tests/pipeline_checks.py @@ -0,0 +1,188 @@ +"""Real DB/S3/Redis fault injection. Stop production worker/dispatcher first.""" +import concurrent.futures +import hashlib +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +sys.path[:0]=['/app/src',str(Path.cwd()/'tests')] +from PIL import Image +from client import Client +from pipeline import STREAM,GROUP,broker,connect,event,dispatch_once,process,ack +from objects import Objects +from postgres import setting +from admin import replay,version,cleanup + + +def main(): + c=Client();alice,bob,eve,chat=c.bootstrap() + bus=broker(); objects=Objects(setting) + report={'invariants':{},'faults':[]} + def fetch(aid): + with connect() as conn: + a=conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s',(aid,)).fetchone() + j=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(a['current_job_id'],)).fetchone() + return a,j + def upload(raw=None,mime='image/png'): + if raw is None: a,_=c.attachment(alice,chat) + else:a=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=research.png',202,actor=alice,raw=raw,headers={'Content-Type':mime},schema='Attachment')['json'] + return a,fetch(a['id'])[1] + def run_once(fault=None): + args=[sys.executable,'/app/src/pipeline.py','work','--once'] + if fault:args+=['--fault-point',fault] + return subprocess.run(args,capture_output=True,timeout=30).returncode + def drain(): + for _ in range(200): + batch=bus.xreadgroup(GROUP,'research-drain',{STREAM:'>'},count=1) + if not batch:break + for mid,data in batch[0][1]:process(json.loads(data['event']));ack(bus,mid) + def expire(job): + with connect() as conn: + conn.execute("UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE 1=1 AND id = %s",(job['id'],)) + def recover(aid): + until=time.monotonic()+20 + while time.monotonic() no publisher yet: one queued row, no Redis message, then automatic outbox pass. + a,j=upload() + assert fetch(a['id'])[0]['state']=='queued' + assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',409,actor=bob) + c.message(bob,chat,'queued is linkable',attachments=[a['id']]) + dispatch_once(bus);drain();assert fetch(a['id'])[0]['state']=='ready' + report['invariants']['commit_before_publish_recovered']=True + # Broker accepted XADD but publisher died before marking outbox: true exit(77). + a,j=upload() + code=subprocess.run([sys.executable,'/app/src/pipeline.py','dispatch','--once','--fault-point','after_publish'],capture_output=True,timeout=10).returncode + assert code==77 + dispatch_once(bus);drain() + assert fetch(a['id'])[0]['state']=='ready' + report['invariants']['publish_before_marker_duplicate_safe']=True + for point in ['before_side_effect','after_original','after_objects','after_commit']: + a,j=upload();dispatch_once(bus) + code=run_once(point);assert code==77,(point,code) + before,job=fetch(a['id']) + if point not in ('after_commit','before_side_effect'):expire(job) + result,job=recover(a['id']) + assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',actor=bob)['bytes']==(Path.cwd()/'tests/fixtures/sample.png').read_bytes() + reclaimed=bus.xautoclaim(STREAM,GROUP,'post-crash-redelivery',min_idle_time=0,start_id='0-0',count=100)[1] + for mid,data in reclaimed: + process(json.loads(data['event']));ack(bus,mid) + assert bus.xpending(STREAM,GROUP)['pending']==0 + keys=objects.client.list_objects_v2(Bucket=objects.bucket,Prefix='results/'+a['id']+'/v1/').get('Contents',[]) + assert len(keys)==4,(point,len(keys)) + report['faults'].append({'point':point,'exit_code':code,'state_at_crash':before['state'],'recovered':result['state'],'attempts':job['attempt'],'result_objects':len(keys)}) + report['invariants']['crashes_before_after_side_effect_and_commit']=True + bus.xadd(STREAM,{'event':json.dumps({'not':'a job'})}) + assert run_once()==0 + assert bus.xlen('rejected-jobs')>=1 + report['invariants']['invalid_envelope_observable_dlq']=True + # Duplicate/reorder through broker on a completed job: no extra attempt/effect. + a,j=upload();dispatch_once(bus);drain();before,done=fetch(a['id']) + old=event(j) + for attempt in [2,1,3,1,1]:bus.xadd(STREAM,{'event':json.dumps(dict(old,attempt=attempt))}) + drain();after,done2=fetch(a['id']) + assert before['variants']==after['variants'] and done2['attempt']==done['attempt'] + report['invariants']['duplicate_and_reorder_no_second_effect']=True + # Race two consumers with the SAME logical job, not just one serial duplicate. + a,j=upload();e=event(j) + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: results=list(pool.map(lambda _:process(e),range(2))) + assert fetch(a['id'])[0]['state']=='ready' and sorted(results)==['noop','ready'] + report['invariants']['concurrent_duplicate_single_claim']=True + # Pause actual old process after all immutable side effects, advance version and finish v2. + a,j=upload();e=event(j) + directory=Path('/tmp')/('fence-'+uuid.uuid4().hex);directory.mkdir();marker=directory/'paused';release=directory/'release' + code="import sys,json,time;from pathlib import Path;sys.path.insert(0,'/app/src');from pipeline import process\ndef hook(point,job):\n if point=='after_objects':\n Path(sys.argv[2]).write_text('paused')\n while not Path(sys.argv[3]).exists():time.sleep(.05)\nprint(process(json.loads(sys.argv[1]),hook))" + child=subprocess.Popen([sys.executable,'-c',code,json.dumps(e),str(marker),str(release)],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + try: + deadline=time.monotonic()+20 + while not marker.exists(): + assert child.poll() is None + assert time.monotonic()=250 + transforms.append({'kind':kind,'size':thumb.size,'exif_removed':not bool(thumb.getexif())}) + report['transforms']=transforms + pending,pending_job=upload() + dry=cleanup(False);fixed=cleanup(True);repeat=cleanup(True) + assert fetch(pending['id'])[0]['state']=='queued' and objects.head(pending_job['source_key']) is not None + report['invariants']['cleanup_preserves_current_and_fresh_failed']=True + assert repeat['counts']['ready_staging']==0 + report['invariants']['ready_staging_cleanup_idempotent']=True + report['cleanup']=[dry,fixed,repeat] + print(json.dumps(report,indent=2)) + + +if __name__=='__main__': main() diff --git a/student-tests/plot.py b/student-tests/plot.py new file mode 100644 index 0000000..157828d --- /dev/null +++ b/student-tests/plot.py @@ -0,0 +1,28 @@ +"""Figures from real accepted/ready and cgroup samples, no synthetic results.""" +import json +from pathlib import Path +import statistics +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +r=Path(__file__).resolve().parents[1]/'evidence' +x=json.loads((r/'resources.json').read_text())['experiments'] +fig,axes=plt.subplots(1,2,figsize=(11,4.3),layout='constrained') +labels=['320×200\n1,3 KiB','1800×1400\n7,21 MiB','10000×4000\n0,51 MiB'] +a=[max(v['accepted_seconds'] for v in e['replies']) for e in x] +b=[max(v['ready_seconds'] for v in e['replies']) for e in x] +axes[0].bar([i-.18 for i in range(3)],a,.36,label='p95 до 202') +axes[0].bar([i+.18 for i in range(3)],b,.36,label='p95 до ready') +axes[0].set(xticks=range(3),xticklabels=labels,ylabel='Секунды',title='10 одновременных загрузок · worker=1') +axes[0].legend() +peak=[int(e['after']['memory.peak'])/1048576 for e in x] +anon=[max(int(dict(line.split() for line in s['worker']['memory.stat'].splitlines())['anon']) for s in e['samples'])/1048576 for e in x] +axes[1].bar([i-.18 for i in range(3)],peak,.36,label='cgroup memory.peak') +axes[1].bar([i+.18 for i in range(3)],anon,.36,label='Пик resident anon') +axes[1].axhline(768,color='#d62728',linestyle='--',label='Лимит worker') +axes[1].set(xticks=range(3),xticklabels=labels,ylabel='MiB',title='Worker + дочерний декодер') +axes[1].legend(fontsize=8) +for ax in axes:ax.grid(axis='y',alpha=.2);ax.set_axisbelow(True) +fig.suptitle('ЛР 5 · реальные измерения · concurrency=1, prefetch=1') +fig.savefig(r/'pipeline.png',dpi=150) +fig.savefig(r/'pipeline.svg') diff --git a/student-tests/plot_observability.py b/student-tests/plot_observability.py new file mode 100644 index 0000000..9c7e956 --- /dev/null +++ b/student-tests/plot_observability.py @@ -0,0 +1,36 @@ +"""Publication plot from measured cost/retention JSON; matplotlib is research-only.""" +import json +from pathlib import Path +import statistics +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +ROOT=Path(__file__).resolve().parents[1] +data=json.loads((ROOT/'evidence/observability-cost.json').read_text()) +modes=['off','full','sample10','sample25'] +labels=['Off','Full 100%','Sample 10%','Sample 25%'] +colors=['#64748b','#2563eb','#0d9488','#d97706'] +fig,axs=plt.subplots(2,3,figsize=(16,9),layout='constrained') +def draw(ax,values,title,ylabel): + med=[statistics.median(v) for v in values] + low=[m-min(v) for m,v in zip(med,values)] + high=[max(v)-m for m,v in zip(med,values)] + ax.bar(range(4),med,color=colors,yerr=[low,high],capsize=4,alpha=.85) + for i,v in enumerate(values):ax.scatter([i]*len(v),v,color='#0f172a',s=14,zorder=3) + ax.set_xticks(range(4),labels,rotation=12) + ax.set_title(title,loc='left',fontweight='bold') + ax.set_ylabel(ylabel) + ax.grid(axis='y',alpha=.2) + ax.set_axisbelow(True) +rows=[[r for r in data['runs'] if r['mode']==mode] for mode in modes] +draw(axs[0,0],[[r['p95_seconds']*1000 for r in group] for group in rows],'HTTP p95','milliseconds') +draw(axs[0,1],[[r['resource']['api']['cpu_seconds'] for r in group] for group in rows],'API CPU (both replicas)','CPU seconds / run + drain') +draw(axs[0,2],[[r['collector_ingest']['otelcol_receiver_accepted_spans_total'] for r in group] for group in rows],'Accepted traces','spans / run + drain') +draw(axs[1,0],[[r['resource']['otel-collector']['rx_bytes']/1024 for r in group] for group in rows],'Collector network received','KiB (includes scrape / protocol traffic)') +draw(axs[1,1],[[r['resource']['api']['peak_sampled_anonymous_bytes']/1048576 for r in group] for group in rows],'API anonymous memory','MiB (sum of 1 Hz sampled replica peaks)') +draw(axs[1,2],[[r['resource']['worker']['cpu_seconds'] for r in group] for group in rows],'Worker CPU incl renderer','CPU seconds / run + drain') +fig.suptitle('Lab 6: cost of observability | 600 mixed requests per run (30 uploads) | 30 req/s | 3 repeats',fontsize=15,fontweight='bold') +fig.savefig(ROOT/'evidence/observability-cost.png',dpi=150) +fig.savefig(ROOT/'evidence/observability-cost.svg') +print('Plot written from actual per-run JSON; dots = repeats, bars = median, whiskers = min/max.') diff --git a/student-tests/resources.py b/student-tests/resources.py new file mode 100644 index 0000000..a9c409d --- /dev/null +++ b/student-tests/resources.py @@ -0,0 +1,52 @@ +"""Finite simultaneous jobs including 40 MP; record actual worker cgroup peak/RSS.""" +import concurrent.futures +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client +from images import png + + +def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip() + + +def memory(container): + code="import json;from pathlib import Path;p=Path('/sys/fs/cgroup');print(json.dumps({k:(p/k).read_text().strip() for k in ['memory.current','memory.peak','memory.events','memory.stat','cpu.stat']}))" + return json.loads(docker('exec',container,'python','-c',code)) + + +def main(): + c=Client(timeout=30);a,b,e,chat=c.bootstrap() + profiles=[(320,200,False),(1800,1400,True),(10000,4000,False)] + result=[] + for w,h,noise in profiles: + content=png(w,h,noise=noise) + docker('compose','restart','worker');container=docker('compose','ps','-q','worker') + before=memory(container);samples=[];done=threading.Event();start=time.monotonic() + def sample(): + while not done.is_set(): + samples.append({'seconds':round(time.monotonic()-start,4),'worker':memory(container)}) + done.wait(.05) + sampler=threading.Thread(target=sample);sampler.start() + barrier=threading.Barrier(10) + def upload(i): + barrier.wait();t=time.monotonic() + r=Client(timeout=30).call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=load.png',202,actor=a,raw=content,headers={'Content-Type':'image/png'},schema='Attachment') + accepted=time.monotonic()-t + ready=Client(timeout=30).wait_ready(b,r['json']) + assert ready['sha256']==hashlib.sha256(content).hexdigest() + return {'index':i,'accepted_seconds':accepted,'ready_seconds':time.monotonic()-t,'status':ready['status']} + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: replies=list(pool.map(upload,range(10))) + finally:done.set();sampler.join() + after=memory(container);assert 'oom_kill 0' in after['memory.events'] + result.append({'width':w,'height':h,'bytes':len(content),'clients':10,'before':before,'after':after,'samples':samples,'replies':replies,'seconds':time.monotonic()-start}) + print(json.dumps({'method':'10 simultaneous uploads, worker concurrency=1; cgroup v2 samples include child renderer; anon is resident anonymous memory, memory.peak includes tmpfs/kernel/cache','experiments':result},indent=2)) + + +if __name__=='__main__':main() diff --git a/student-tests/sampling_retention.py b/student-tests/sampling_retention.py new file mode 100644 index 0000000..a8e7dda --- /dev/null +++ b/student-tests/sampling_retention.py @@ -0,0 +1,38 @@ +"""Separate synchronized rare-error/slow-success retention experiment.""" +import json +from pathlib import Path +from observability_cost import set_mode, probes +from observability import save, trace, wait +import urllib.error + + +def verify_presence(rows): + for row in rows: + if row["trace_sampled"]: + data=wait(lambda:trace(row["trace_id"]),timeout=90) + assert data.get("batches",data.get("resourceSpans")) + row["tempo_trace_found"]=True + else: + try: + trace(row["trace_id"]) + except urllib.error.HTTPError as error: + assert error.code==404 + row["tempo_trace_found"]=False + else: + raise AssertionError("Unsampled trace unexpectedly stored") + return rows + +def main(): + result={'method':'Three repeats per full/10%/25% setting; each holds sessions lock until pg_locks confirms it. 3x503 plus one slow200 per repeat. Sampling is ordinary SDK head sampling without forced traceparent.','runs':[]} + try: + for mode,ratio in [('full',1),('sample10',.1),('sample25',.25)]: + c=set_mode(1,ratio) + actor=c.user('retention') + for repeat in range(3): + result['runs'].append({'mode':mode,'repeat':repeat,'probes':verify_presence(probes(c,actor,mode))}) + save('sampling-retention.json',result) + print(mode,repeat,'retention recorded',flush=True) + finally:set_mode(1,.25) + print('PASS: synchronized rare 503/slow200 retention study; balanced sampling restored.') + +if __name__=='__main__':main() diff --git a/student-tests/security.py b/student-tests/security.py new file mode 100644 index 0000000..7fb5321 --- /dev/null +++ b/student-tests/security.py @@ -0,0 +1,29 @@ +"""Private S3 and job envelopes do not expose source bytes or credentials.""" +import json +from pathlib import Path +import sys +import urllib.error +import urllib.request +sys.path[:0]=['/app/src',str(Path.cwd()/'tests')] +from client import Client +from pipeline import connect,validate,event +from objects import Objects +from postgres import setting + +c=Client();a,b,e,chat=c.bootstrap();att,raw=c.attachment(a,chat);ready=c.wait_ready(b,att) +with connect() as conn: + row=conn.execute('SELECT object_key, current_job_id FROM attachments WHERE 1=1 AND id = %s',(att['id'],)).fetchone() + job=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(row['current_job_id'],)).fetchone() +message=event(job);message['attempt']=max(1,job['attempt']) +assert validate(message) +required={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'} +assert required <= set(message) <= required | {'traceparent','tracestate'} +assert validate(message) +assert len(json.dumps(message))<1024 +for path in ('/'+Objects(setting).bucket+'/'+row['object_key'],'/'+Objects(setting).bucket+'?list-type=2'): + try:urllib.request.urlopen('http://s3:8333'+path,timeout=2);raise AssertionError('anonymous access') + except urllib.error.HTTPError as error:assert error.code==403 +for key in ['schema_version','pipeline_version','attempt']: + bad=dict(message);bad[key]=True;assert not validate(bad) +for body in [dict(message,extra='x'),dict(message,created_at='bad'),dict(message,traceparent='bad')]:assert not validate(body) +print(json.dumps({'anonymous_object_and_list_403':True,'job_fields':sorted(message),'job_bytes':len(json.dumps(message)),'no_source_bytes_or_credentials_in_envelope':True,'strict_invalid_envelope_checks':True},indent=2)) diff --git a/student-tests/slow_upload.py b/student-tests/slow_upload.py new file mode 100644 index 0000000..c17d2b0 --- /dev/null +++ b/student-tests/slow_upload.py @@ -0,0 +1,32 @@ +"""A continuously trickling client still reaches the absolute upload deadline.""" +import json +from pathlib import Path +import select +import socket +import sys +import time +import uuid +from urllib.parse import urlsplit +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +c = Client() +alice, bob, eve, chat = c.bootstrap() +target = urlsplit(c.base) +connection = socket.create_connection((target.hostname, target.port), timeout=3) +header = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=slow_' + uuid.uuid4().hex + '.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n' +started = time.monotonic() +connection.sendall(header.encode()) +response = b'' +while time.monotonic() - started < 20: + if select.select([connection], [], [], .15)[0]: + response += connection.recv(4096) + if b'\r\n\r\n' in response: + break + connection.sendall(b'x') +connection.close() +elapsed = time.monotonic() - started +assert response.startswith(b'HTTP/1.1 400') or response.startswith(b'HTTP/1.0 400'), response[:100] +assert 14 <= elapsed < 19 +assert b'x-instance-id:' in response.lower() and b'x-request-id:' in response.lower() +print(json.dumps({'trickle_seconds': round(elapsed, 3), 'http_status': 400, 'absolute_deadline_enforced': True})) diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..45f0d80 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,177 @@ +"""Extra black-box tests, without changing the course's public tests.""" +import concurrent.futures +from pathlib import Path +import sys +import unittest +from urllib.parse import quote +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client as PublicClient, LAB + + +class Client(PublicClient): + def call(self, method, path, *args, headers=None, **kwargs): + headers = dict(headers or {}) + if method == 'POST' and path.endswith('/messages'): + headers.setdefault('Idempotency-Key', uuid.uuid4().hex) + return super().call(method, path, *args, headers=headers, **kwargs) + + +class APIContract(unittest.TestCase): + def setUp(self): + self.client = Client() + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.path = '/api/v1/chats/' + self.chat['id'] + + def test_malformed_json_and_extra_fields(self): + for raw in (b'{', b'[]', b'{"text":"x","text":"y"}', b'{"text":NaN}', b'{"text":"\\ud800"}'): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'}) + for body in ({'text': 'ok', 'sender_id': self.eve['id']}, {'text': True}, {'text': ' '}, {'text': ''}, {'text': None}): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body=body) + deep = b'{"text":' + b'[' * 1200 + b'0' + b']' * 1200 + b'}' + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=deep, headers={'Content-Type': 'application/json'}) + + def test_unicode_limits(self): + result = self.client.message(self.alice, self.chat, '👩' * 4000)['json'] + self.assertEqual(result['text'], '👩' * 4000) + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': '👩' * 4001}) + + def test_unicode_nul_round_trip(self): + import base64 + value = 'A\x00B\\u0000👩' + registration = dict(self.alice['registration'], username='nul_' + uuid.uuid4().hex[:20], display_name=value) + user = self.client.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json'] + self.assertEqual(user['display_name'], value) + credential = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode() + session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json'] + self.assertEqual(session['user'], user) + actor = {'id': user['id'], 'token': session['token']} + # Exercise the database path and then the Redis profile cache. + for _ in range(2): + self.assertEqual(self.client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'], user) + chat = self.client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': value, 'member_ids': [actor['id']]}, schema='Chat')['json'] + self.assertEqual(chat['title'], value) + self.assertEqual(self.client.call('GET', '/api/v1/chats/' + chat['id'], actor=actor, schema='Chat')['json'], chat) + contents = [value, '\x00' * 4000, '👩' * 3999 + '\x00'] + for content in contents: + self.assertEqual(self.client.message(actor, chat, content)['json']['text'], content) + history = self.client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor, schema='MessagePage')['json'] + self.assertEqual([message['text'] for message in history['items']], contents) + + def test_foreign_chat(self): + for suffix in ('', '/messages'): + self.client.call('GET', self.path + suffix, 404, actor=self.eve) + self.client.call('POST', self.path + '/messages', 404, actor=self.eve, body={'text': 'hidden'}) + self.client.call('GET', '/api/v1/chats/' + str(uuid.uuid4()), 404, actor=self.alice) + + def test_registration_race(self): + body = dict(self.alice['registration'], username='race_' + uuid.uuid4().hex[:20]) + def register(_): + return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status'] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + statuses = list(executor.map(register, range(10))) + self.assertEqual(statuses.count(201), 1) + self.assertEqual(statuses.count(409), 9) + + def test_invalid_chat_creation(self): + members = ([self.alice['id'], self.alice['id']], [self.bob['id']], [self.alice['id'], str(uuid.uuid4())], [], [True]) + for member_ids in members: + self.client.call('POST', '/api/v1/chats', 400, actor=self.alice, body={'title': 'rollback', 'member_ids': member_ids}) + + def test_cursor_scope_and_validation(self): + for i in range(3): + self.client.message(self.alice, self.chat, str(i)) + cursor = self.client.call('GET', self.path + '/messages?limit=1', actor=self.alice, schema='MessagePage')['json']['next_cursor'] + other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'other', 'member_ids': [self.alice['id']]}, schema='Chat')['json'] + self.client.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(cursor), 400, actor=self.alice) + for query in ('limit=0', 'limit=101', 'limit=' + '9' * 5000, 'limit=true', 'limit=1&limit=2', 'cursor=', 'cursor=bad', 'cursor=e30=', 'offset=1'): + self.client.call('GET', self.path + '/messages?' + query, 400, actor=self.alice) + + def test_concurrent_messages_and_pagination(self): + def send(i): + return Client().message(self.alice if i % 2 else self.bob, self.chat, 'parallel-' + str(i))['json'] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + sent = list(executor.map(send, range(100))) + found = [] + cursor = None + while True: + url = self.path + '/messages?limit=7' + ('&cursor=' + quote(cursor) if cursor else '') + page = self.client.call('GET', url, actor=self.alice, schema='MessagePage')['json'] + found.extend(page['items']) + cursor = page['next_cursor'] + if cursor is None: + break + self.assertEqual(len(found), 100) + self.assertEqual(len({v['id'] for v in found}), 100) + self.assertEqual({v['id'] for v in found}, {v['id'] for v in sent}) + self.assertEqual(found, sorted(found, key=lambda v: (v['created_at'], v['id']))) + + def test_idempotency_concurrent_replicas(self): + key = uuid.uuid4().hex + def send(_): + return Client().message(self.alice, self.chat, 'Один раз\x00👩', key=key) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + replies = list(executor.map(send, range(32))) + self.assertGreaterEqual(len({v['headers']['X-Instance-Id'] for v in replies}), 2) + self.assertTrue(all(v['json'] == replies[0]['json'] for v in replies)) + page = self.client.call('GET', self.path + '/messages', actor=self.bob, schema='MessagePage')['json'] + self.assertEqual(page['items'], [replies[0]['json']]) + self.client.call('POST', self.path + '/messages', 409, actor=self.alice, body={'text': 'different'}, headers={'Idempotency-Key': key}) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=self.alice) + self.client.call('POST', self.path + '/messages', 401, actor=self.alice, body={'text': 'Один раз\x00👩'}, headers={'Idempotency-Key': key}) + + def test_protocol_error_headers(self): + self.client.call('TRACE', '/api/v1/users/me', 400, actor=self.alice) + + def test_idempotency_scope(self): + key = uuid.uuid4().hex + one = self.client.message(self.alice, self.chat, 'same', key=key)['json'] + two = self.client.message(self.bob, self.chat, 'same', key=key)['json'] + other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'scope', 'member_ids': [self.alice['id']]}, schema='Chat')['json'] + three = self.client.message(self.alice, other, 'same', key=key)['json'] + self.assertEqual(len({one['id'], two['id'], three['id']}), 3) + + def test_idempotency_semantic_body(self): + key = uuid.uuid4().hex + one = self.client.message(self.alice, self.chat, 'semantic', key=key)['json'] + two = self.client.call('POST', self.path + '/messages', 201, actor=self.alice, body={'attachment_ids': [], 'text': 'semantic'}, headers={'Idempotency-Key': key}, schema='Message')['json'] + self.assertEqual(one, two) + + def test_idempotency_invalid_keys(self): + for key in (None, '', 'x' * 129, 'space key', 'bad,key', 'slash/key'): + headers = {} if key is None else {'Idempotency-Key': key} + PublicClient().call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': 'valid'}, headers=headers) + for raw in (b'{"text":true}', b'{"text":"ok","extra":1}', b'{"text":"ok","attachment_ids":false}'): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'}) + + def test_idempotency_conflicting_race(self): + key = uuid.uuid4().hex + def send(i): + return Client().call('POST', self.path + '/messages', [201, 409], actor=self.alice, body={'text': str(i % 2)}, headers={'Idempotency-Key': key}, schema=None) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + replies = list(executor.map(send, range(32))) + self.assertEqual(sum(v['status'] == 201 for v in replies), 16) + self.assertEqual(sum(v['status'] == 409 for v in replies), 16) + page = self.client.call('GET', self.path + '/messages', actor=self.alice, schema='MessagePage')['json'] + self.assertEqual(len(page['items']), 1) + + @unittest.skipUnless(LAB >= 2, 'sessions start in lab2') + def test_unicode_password_basic_and_logout(self): + import base64 + registration = {'username': 'unicode_' + uuid.uuid4().hex[:16], 'display_name': 'Софья', 'password': '🔐' * 127 + ':'} + self.client.call('POST', '/api/v1/users', 201, body=registration) + encoded = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode() + headers = {'Authorization': 'Basic ' + encoded} + session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers=headers, schema='Session')['json'] + actor = {'id': session['user']['id'], 'token': session['token']} + self.client.call('GET', '/api/v1/users/me', actor=actor) + self.client.call('GET', '/api/v1/users/me', 401, headers=headers) + for authorization in ('Basic !!!', 'Bearer wrong', '', 'Basic Zm9v'): + self.client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': authorization}) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor) + self.client.call('GET', '/api/v1/users/me', 401, actor=actor) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/student-tests/test_images.py b/student-tests/test_images.py new file mode 100644 index 0000000..f27c039 --- /dev/null +++ b/student-tests/test_images.py @@ -0,0 +1,52 @@ +"""Independent tests for async image metadata, transformations and access.""" +import io +from pathlib import Path +import sys +import unittest +import uuid +from urllib.parse import quote +sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'tests')] +from client import Client +from images import bomb + + +class Images(unittest.TestCase): + def setUp(self): + self.c=Client(); self.alice,self.bob,self.eve,self.chat=self.c.bootstrap() + self.path='/api/v1/chats/'+self.chat['id'] + self.raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes() + + def test_async_variants_and_replay(self): + queued,_=self.c.attachment(self.alice,self.chat) + self.assertEqual(queued['status'],'queued') + key=uuid.uuid4().hex + one=self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json'] + self.assertEqual(self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json'],one) + ready=self.c.wait_ready(self.alice,queued) + self.assertEqual({v['name'] for v in ready['variants']},{'thumbnail','crop','optimized'}) + dimensions={v['name']:(v['width'],v['height']) for v in ready['variants']} + self.assertEqual(dimensions,{'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)}) + for v in ready['variants']: + import hashlib + r=self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant='+v['name'],actor=self.bob) + self.assertEqual(hashlib.sha256(r['bytes']).hexdigest(),v['sha256']) + self.assertEqual(len(r['bytes']),v['size_bytes']) + self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant=absent',404,actor=self.bob) + + def test_mime_bytes_pixels_and_filename(self): + path=self.path+'/attachments?filename=test.png' + for raw,mime,status in [(b'not image','image/png',415),(self.raw[:20],'image/png',415),(self.raw,'image/jpeg',415),(bomb(),'image/png',413),(b'x'*(10485760+1),'image/png',413)]: + self.c.call('POST',path,status,actor=self.alice,raw=raw,headers={'Content-Type':mime}) + for name in ('../x.png','a\\x.png','a\0.png','a\n.png','a'*256,''): + self.c.call('POST',self.path+'/attachments?filename='+quote(name,safe=''),400,actor=self.alice,raw=self.raw,headers={'Content-Type':'image/png'}) + + def test_private_and_cross_chat(self): + att,_=self.c.attachment(self.alice,self.chat) + for suffix in ('','/content'): + self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=self.eve) + self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,401) + other=self.c.call('POST','/api/v1/chats',201,actor=self.alice,body={'title':'other','member_ids':[self.alice['id']]},schema='Chat')['json'] + self.c.call('POST','/api/v1/chats/'+other['id']+'/messages',404,actor=self.alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + + +if __name__=='__main__': unittest.main(verbosity=2)