ЛР 6 — БВТ2301 — Майорова — на 5 #52

Open
mayorovasonya wants to merge 1 commits from mayorovasonya/mtusi2026lab6:solution/BVT2301-Mayorova into main
80 changed files with 22966 additions and 50 deletions
+2
View File
@@ -18,3 +18,5 @@ __pycache__
node_modules
evidence
.DS_Store
build
+16 -27
View File
@@ -1,30 +1,19 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
APP_PORT=18093
GRAFANA_PORT=13006
PROMETHEUS_PORT=19096
LOKI_PORT=13106
TEMPO_PORT=13206
COLLECTOR_PORT=18896
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
API_MEMORY=256m
HTTP_WORKERS=16
DB_POOL_MAX=8
DB_ACCESS_MODE=optimized
WORKER_CONCURRENCY=1
MAX_PENDING_JOBS=100
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
JOB_TIMEOUT_SECONDS=20
JOB_LEASE_SECONDS=60
TELEMETRY_ENABLED=1
TRACE_SAMPLE_RATIO=0.25
+24
View File
@@ -0,0 +1,24 @@
name: Application
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 15
env:
BASE_URL: http://127.0.0.1:18093
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- run: python3 scripts/bootstrap.py
- run: docker compose build api
- run: docker compose up -d --wait
- run: python3 scripts/wait_ready.py
- run: make check test test-student BASE_URL=$BASE_URL
- run: make test-observability BASE_URL=$BASE_URL
- run: make test-persistence BASE_URL=$BASE_URL ACTION_SCRIPT=scripts/restart.sh
- if: always()
run: docker compose down -v
+13
View File
@@ -0,0 +1,13 @@
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 MALLOC_ARENA_MAX=2
WORKDIR /app
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt
COPY src/ ./src/
COPY migrations/ ./migrations/
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"]
STOPSIGNAL SIGTERM
CMD ["python", "src/server.py"]
+16
View File
@@ -28,3 +28,19 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: test-student
test-student:
BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p 'test_*.py' -v
.PHONY: test-observability test-incidents test-telemetry-outage
test-observability:
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py proof
test-incidents:
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py incidents
test-telemetry-outage:
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py backend
.PHONY: research
research:
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability_cost.py
+336 -23
View File
@@ -1,37 +1,350 @@
# Отчёт — лабораторная 6
# Лабораторная 6 — наблюдаемость мессенджера
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
**Майорова Софья, БВТ2301. Целевая оценка: 5, трек 1 «Цена наблюдаемости».**
Дата выполнения: 20 ноября 2026 года. Оценку подтверждает преподаватель.
Продолжена [собственная lab5, PR 7](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/7).
Перенесены только исходники, миграции, свои тесты и необходимые конфигурации.
README/COURSE/contracts/tests текущего skeleton lab6 сохранены, assertions публичных проверок не менялись.
Трек 2 с multi-window burn rate не заявляется.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Docker Engine/Compose v2, Linux-контейнеры, Python 3.9+ для host-проверок.
Сервер и worker Python 3.12.8 внутри Docker. На этом Mac:
## Архитектура и решения
```sh
colima start --profile mtusi-labs
export DOCKER_CONTEXT=colima-mtusi-labs
cd "/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab6"
python3 scripts/bootstrap.py
docker compose build api
docker compose up -d --wait
export BASE_URL=http://127.0.0.1:18093
python3 scripts/wait_ready.py
make check test test-student BASE_URL=$BASE_URL
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
На другой машине клонировать публичный fork, выбрать solution/BVT2301-Mayorova и выполнять из корня clone.
Образ приложения сначала строится через build api: worker, dispatcher, migration и audit используют тот же image.
Bootstrap генерирует локальные случайные secrets, сохраняет существующие значения и .env.
Каталог secrets 0700; полный S3 identity config 0600, монтируется только root gateway.
Пароли, ключи, .env, volumes, backups и build не коммитятся и не попадают в build context.
## Доказательства критериев
API: http://127.0.0.1:18093.
Grafana: http://127.0.0.1:13006/d/messenger-lab6 — автоматический Viewer на localhost.
GF_USERS_VIEWERS_CAN_EDIT разрешает Explore и временные правки запросов без сохранения dashboard;
роль остаётся Viewer (настройка deprecated в Grafana 12.1.1, доступна в закреплённой версии).
Импорт dashboard и трёх datasources выполняет provisioning из Git; ручной настройки не требуется.
Prometheus http://127.0.0.1:19096, Loki 13106, Tempo 13206, собственные метрики Collector 18896.
OTLP 4318 не публикуется на host. PostgreSQL/S3/broker/API-реплики тоже не публикуются в обход LB.
Операторские UI присоединены к edge для корректного loopback port mapping Docker; backend и storage — внутренние сети.
Наружу в Интернет эти порты не открыты. Grafana не является административной панелью бизнес-данных.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Остановка с сохранением истории: docker compose down.
Не удалять volumes в проверке restart/failover. down -v предназначен только для намеренно уничтожаемого отдельного тестового стенда.
На этой машине старый Docker proxy не разрешался DNS.
После отдельного разрешения пользователя сохранён backup /etc/docker/daemon.json.lab6-proxy-backup внутри VM,
удалена неработающая proxy-секция и перезапущен Docker daemon. Реестр проверен по прямому HTTPS.
Это локальная настройка окружения, не часть приложения и не обязательный шаг для другого clone.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Архитектура
## Эксперименты и отказы
```text
Клиент → HAProxy → API ×2 → PostgreSQL
├→ Redis cache профилей
└→ приватный S3 staging
PostgreSQL jobs/outbox → dispatcher → Redis Streams/AOF → worker → S3 results + DB ready
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
API / dispatcher / worker → OpenTelemetry SDK → OTLP HTTP/protobuf → Collector
├→ metrics → Prometheus → Grafana
├→ logs → Loki → Grafana Explore
└→ traces → Tempo → Grafana Explore
```
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Prometheus хранит временные ряды, Loki — структурные логи, Tempo — трассы.
Grafana выполняет запросы и визуализирует; Collector принимает/обрабатывает/экспортирует и не заменяет долговременные хранилища.
PostgreSQL остаётся источником истины бизнес-данных и сессий. Наблюдаемость не участвует в COMMIT подтверждённого сообщения/задания.
## Ограничения и следующие шаги
Сохранён минимум предыдущих работ: scrypt и сессии с expiry/logout, Unicode/U+0000, ACL чатов,
keyset, атомарный Idempotency-Key на 24h, две реплики и migration job, Redis profile cache,
приватные изображения, durable staging/outbox, bounded renderer, at-least-once/retry/lease,
immutable versioned results, version fencing, failed registry/replay и операторский cleanup.
API возвращает 202 queued после S3 staging и COMMIT; resize выполняет отдельный worker.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Миграции 001–006. Новая 006 добавляет trace_context JSONB к attachment_jobs.
Миграция выполняется отдельным service под advisory lock; API проверяет наличие 006 до обслуживания.
Старые jobs получают пустой carrier и продолжают работать; бизнес-API не расширялся.
## Версии и ресурсы
Все Docker images закреплены version + digest в Compose/Dockerfile.
Python 3.12.8; PostgreSQL 17.2; Redis 7.4.1; SeaweedFS 4.51; HAProxy 3.0.5.
OTel Python API/SDK/HTTP exporter 1.36.0, semantic conventions 0.57b0;
Collector contrib 0.132.0; Prometheus 3.5.0; Loki 3.5.3; Tempo 2.8.2; Grafana 12.1.1.
Все 27 Python distributions, включая транзитивные, закреплены в requirements.txt по успешной сборке.
API ×2: по 1 CPU/256MiB; worker 1 CPU/768MiB, concurrency1; dispatcher 1 CPU/256MiB.
PostgreSQL 1 CPU/512MiB; broker0.5CPU/192MiB/AOF always; Redis cache0.25CPU/64MiB.
S3 store1CPU/512MiB и gateway0.5CPU/512MiB; LB0.5CPU/64MiB.
Collector/Prometheus/Loki/Grafana — по 0.5CPU/256MiB, Tempo0.5CPU/512MiB.
Сумма CPU-лимитов — overcommit, не выделенные физические ядра.
Окружение: macOS arm64, Colima Linux VM4vCPU/около4GiB; локальный synthetic workload.
## Схема сигналов и единицы
Ручная SDK-инструментация ограничивает атрибуты разрешённым набором.
Resource: service.name=messenger-api / messenger-worker / messenger-dispatcher;
service.version=lab6.1; service.instance.id — случайный alias процесса;
deployment.environment.name=lab. Дополнительно SDK сообщает собственную версию/язык.
| Реальное имя Prometheus | Значение |
| --- | --- |
| http_server_requests_total | Обработанные business requests; method, route template, status_class |
| http_server_duration_seconds_bucket/count/sum | Полная обработка HTTP, включая отправку ответа |
| jobs_processed_total | Попытки worker: ready, failed_or_retry, noop, fenced, error; job_type=image |
| job_processing_duration_seconds_bucket/count/sum | Время попытки, тот же outcome |
| job_ready_duration_seconds_bucket/count/sum | От created_at durable job перед COMMIT до публикации ready |
| job_retries_total | Решения retry/terminal failed, outcome=retry или failed |
| queue_depth | Незавершённые queued/processing в PostgreSQL, включая outbox до broker |
| queue_oldest_age_seconds | Возраст самой старой такой job; backlog не processing latency |
| queue_observation_ok | 0 при невозможности наблюдать PostgreSQL backlog |
| db_pool_used/waiting/capacity | Используемые/ожидающие/максимум соединений API |
| worker_busy/capacity | Активные попытки/число slots |
| process_cpu_seconds_total | CPU процесса, cumulative |
| process_resident_memory_bytes | RSS процесса через /proc/self/statm |
| container_memory_bytes | Cgroup memory.current, включая renderer и cache |
| spans_finished/exported/export_failed | Число sampled SDK spans и результат отправки |
В gauge количества unit пустой: иначе unit1 экспортируется Prometheus как _ratio.
Очередь именуется постоянным attachment-jobs; отдельного динамического queue label не нужно.
Service instance — resource attribute с ограниченной жизнью процесса; churn реплик всё равно создаёт новые series.
Health полностью исключён из business metrics/logs/traces. Не распознанные маршруты имеют template unmatched.
user/chat/message/attachment/request/job/trace IDs, filename, raw URL и текст не являются labels метрик.
Границы duration histogram, секунды:
0.001,0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2,5,10,20,30,60,+Inf.
Dashboard агрегирует buckets всех реплик:
histogram_quantile(0.95, sum by(le)(rate(http_server_duration_seconds_bucket[1m]))).
p50/p99 построены аналогично; средние p95 реплик не усредняются.
Точность percentile ограничена границами buckets и количеством наблюдений.
## Трассы, логи и очередь
Handler создаёт SERVER span с method/route/status без headers/body/query.
Connection context создаёт CLIENT db.transaction span, включающий ожидание пула, SQL и завершение транзакции,
но без текста SQL, параметров, connection string или сырого текста exception.
Это время unit of work, а не чистый server-side SQL execution time: S3 PUT под transaction виден дочерним S3 span.
S3 get span заканчивается при получении SDK response; потоковое чтение Body входит во время родительского HTTP/worker span.
S3 wrapper записывает только service S3 и имя операции; ключи, filenames, URLs и credentials не записываются.
Worker consumer span включает processing и реальные дочерние S3/DB spans.
Внутри изолированного renderer отдельный SDK не запускается.
W3C traceparent/tracestate сохраняются с job в той же транзакции outbox.
Dispatcher извлекает durable context, создаёт PRODUCER span и вкладывает его carrier в envelope.
Worker извлекает envelope и создаёт CONSUMER child producer. Retry публикуется с тем же исходным trace,
новым producer/consumer и номером попытки. Envelope содержит ID/версию/попытку/carrier, а не bytes изображения.
Структурный JSON log содержит timestamp, severity, service.name, event, trace_id, span_id, sampled;
HTTP — X-Request-Id, template/method/status/duration; job — технический job_id/attempt/outcome.
Эти технические ID нужны для сквозной диагностики синтетического задания и не индексируются как metric labels.
LoggingHandler отправляет OTLP logs; Loki принимает native OTLP и хранит остальные поля как structured metadata/body.
Grafana derived field TraceID ведёт из Loki в Tempo; tracesToLogs фильтрует тот же trace_id.
Имена полей/шаблоны фиксированы. Тела сообщений, картинки, пароли, Authorization, tokens и signed URLs исключены.
Проверка proof выполняет реальный upload с известным traceparent, ждёт ready,
проверяет родителя consumer=producer и цепочку предков producer до SERVER upload,
проверяет worker S3/DB spans и логи всех стадий, связывает response X-Request-Id.
Отдельно остановлен worker, принят файл, приостановлен реальный S3 gateway, worker запущен:
наблюдается S3 timeout/retry; после возврата S3 второй consumer достигает ready в той же трассе.
Canary message/password/token/username/filename не найдены в проверяемом trace и app logs с начала controlled proof (лимит 5000);
это ограниченная динамическая проверка плюс allowlist кода, не математическое доказательство отсутствия любых утечек.
Метрики проверены на запрещённые ID labels.
## SLI/SLO и алерты
Полные определения/ограничения — infra/RUNBOOK.md.
Окно5мин: API availability>=99% обработанных business requests без5xx; health исключён,
4xx не смешиваются с отказами сервера. Отказ LB/сети до API данным SLI не покрывается.
Latency SLO: >=95% завершённых ready jobs укладываются в5сек от durable job creation.
Это условный latency SLO, не доступность pipeline; queued/failed отдельно видны через backlog/outcomes/alerts.
Prometheus правила из infra/alerts.yaml, evaluation2s:
- APIHighErrorRate: 5xx share за30s>5%, for6s.
- QueueLag: oldest>10s и depth>0, for6s.
- QueueObservationUnavailable: observer0, for6s.
Это короткие учебные operational alerts, не трек2 multi-window burn rate.
Нет email/Slack или внешней доставки.
## Два инцидента и диагностика
1. Worker остановлен, API принял3 изображения.
Queue depth/oldest растут, завершения worker прекращаются; upload202 и логи acceptance/publish продолжаются.
QueueLag firing примерно через22.77сек. После возврата worker все три ready, backlog0 и alert resolved.
Значения gauges остановленного процесса могут оставаться в scrape cache до30s: это явно учитывается в runbook.
2. Реальная ACCESS EXCLUSIVE блокировка sessions удерживается35s.
Readiness SELECT1 остаётся зелёной, защищённый endpoint ждёт SQL и получает503 по lock_timeout1.5s.
APIHighErrorRate firing примерно через13.14сек. В сохранённой трассе db.transaction — долгий/error span;
это отделяет ожидание PostgreSQL от HTTP handler/resize. После COMMIT блокировки прежний Bearer снова работает,
30s error window очищается и alert resolved.
Сырые timestamps, firing snapshots, запросы метрик, error latencies и DB trace — evidence/incidents.json.
Разбор опирается на наблюдения; способ инъекции задокументирован отдельно и не заменяет trace/metric evidence.
## Отказ telemetry backend и конечная буферизация
Tempo остановлен примерно на56сек. За это время API подтвердил135 сообщений и5 файлов,
worker завершил файлы, все подтверждённые сообщения найдены в истории.
Collector memory snapshots: примерно58.21–62.05MiB при лимите256MiB; наблюдены156 failed exported spans.
После восстановления найден новый реальный trace. Потерянная telemetry не пересоздаётся из бизнес-данных.
SDK: BatchSpanProcessor queue512/batch128, logs queue256/batch64, schedule500ms,
HTTP timeout1s; metric export каждые2s. Отправка асинхронная, очереди ограничены.
Collector memory_limiter limit192/spike32MiB, batch<=256;
два exporter queue по128 batches,2consumers, retry budget10s.
Prometheus exporter имеет expiration30s. Нет безразмерной очереди или бесконечного retry.
Лимит batches не является точным лимитом байт; SDK/receiver имеют дополнительные ограничения,
container memory лимит — последняя граница. Локальный тест конечной длительности не доказывает поведение при любом трафике.
Метрики хранятся24h и ограничены256MB TSDB; Loki/Tempo retention24h; Grafana SQLite/provisioning.
Именованные metrics_data/logs_data/traces_data/grafana_data сохраняются при restart.
Loki/Tempo retention выполняется асинхронно и не является жёсткой дисковой квотой.
## Исследование цены наблюдаемости
Гипотеза: SDK export/sampling влияет на задержки, CPU/RAM и ingest; уменьшение доли трасс сокращает ingest,
но часть редких ошибок и медленных запросов не имеет полного trace.
Методика student-tests/observability_cost.py:
off, full1.0, head sampling0.10 и0.25; три повтора, порядок режимов циклически сдвигается.
Одинаковые лимиты,2API,worker concurrency1, fixture320x200.
Каждый прогон: фиксированные30 scheduled requests/s20s,600 запросов:
5% upload,15% messages,80% profile reads;16client threads.
Exporter off оставляет установленные SDK/hooks, но не создаёт providers и не экспортирует signals.
Это не сравнение с переписанной версией без SDK imports.
Измеряются клиентские p50/p95/p99, подтверждённый RPS/ошибки,
CPU cgroup API/worker/dispatcher/Collector/хранилищ,
anonymous/working-set memory samples1Hz,
точные counter deltas принятых spans/logs/metric points и Collector receive network bytes.
Последние включают OTLP, Prometheus scrape и backend replies: это не чистый размер protobuf payload.
CPU/ingest включают ожидание exporter drain5s и проверки ready после основного потока.
История БД/telemetry сохраняется между режимами; новые API/worker/dispatcher создаются на каждом режиме.
Пять секунд прогрева/drain, фиксированная среда и смена порядка уменьшают, но не исключают шум/влияние кэша.
UUID/sampling стохастические; нагрузка локальная и короткая. Это не capacity ceiling и не production SLA.
Первоначальные SQL-lock probes после измерительных окон не синхронизировали момент HTTP с pg_locks:
часть блокировок успевала завершиться при старте docker exec. Их фактические HTTP статусы сохранены
в rare_probes основного raw JSON; выдавать их за гарантированные 503 нельзя.
Для вывода о редких событиях выполнено отдельное исследование student-tests/sampling_retention.py:
full/10%/25%, три повтора; в каждом три настоящих503 и один медленный200.
Блокировки3.5s/1.2s проверяются через granted pg_locks перед HTTP, status и duration проверяются assertions.
У каждого режима получено9ошибок и3медленных успешных запроса.
### Измеренные результаты
Всего12основных прогонов,7200/7200 подтверждённых запросов без ошибок,360/360 изображений ready.
Ниже медиана трёх прогонов; ranges и каждый raw result доступны в JSON, точки/отрезки на графике.
CPU — накопленные секунды двух API или worker за измерительное окно с drain, не проценты host CPU.
| Режим | HTTP p95, ms | p99, ms | API CPU, s | Worker CPU, s | Collector CPU, s | Принятые spans | Collector RX, KiB |
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| off | 9.818 | 15.301 | 2.371 | 5.464 | 0.054 | 0 | 13.2 |
| full | 9.695 | 14.553 | 2.799 | 5.533 | 0.194 | 1950 | 1067.0 |
| 10% | 9.149 | 13.731 | 2.541 | 5.270 | 0.167 | 176 | 720.9 |
| 25% | 10.557 | 15.699 | 2.618 | 5.220 | 0.183 | 467 | 802.8 |
| Режим | Сумма sampled anon peaks двух API, MiB | Collector anon peak, MiB | Logs / metric points, медиана |
| --- | ---: | ---: | ---: |
| off | 159.60 | 56.02 | 0 / 0 |
| full | 147.90 | 56.03 | 840 / 1011 |
| 10% | 168.66 | 58.78 | 840 / 987 |
| 25% | 147.82 | 58.72 | 840 / 1011 |
Сумма отдельных sampled peaks не является одновременным peak всей группы; интервалы memory/p95 перекрываются.
Результаты не доказывают ускорение запросов или экономию RAM. Разница CPU небольшая и подвержена шуму.
Full даёт примерно18% больше API CPU, чем off, в медианах этого опыта.
25% уменьшает принятые spans на76.1%, сетевой RX Collector на24.8% и API CPU примерно на6.5% относительно full.
Ingest логов/метрик сохраняется независимо от trace sampling, поэтому суммарный трафик не падает на75%.
Head sampling применяется к целой цепочке: число дочерних spans в sampled image jobs меняет фактическую долю spans.
После drain sampled spans SDK и accepted spans Collector совпали во всех12прогонах:
дополнительная потеря между SDK и Collector в данном измерении0; несэмплированные spans в это число не входят.
| Режим | Сохранён trace ошибок503 | Сохранён trace медленных200 |
| --- | ---: | ---: |
| full | 9/9 | 3/3 |
| 10% | 0/9 | 1/3 |
| 25% | 1/9 | 1/3 |
Это наблюдения малой выборки, а не оценка вероятности или гарантия. Логи этих HTTP и полные метрики
сохраняются; sampled проверяется в коррелированном HTTP log;
для всех36probes также проверено реальное наличие sampled trace и404 для unsampled через Tempo API. Head sampling принимает решение до результата,
поэтому не может гарантировать trace ошибки. Для расследования воспроизводимого сбоя временно выбирать1.0
или отправлять контролируемый sampled W3C parent (как proof); это не автоматическое сохранение всех ошибок.
По умолчанию выбран25% как компромисс объёма трасс и покрытия при сохранённых logs/metrics.
10% даёт ещё меньший ingest, но более редкие полные причинные цепочки. В режиме incident диагностики подходит full.
Отдельный повтор выбранного режима:600/600 запросов,30/30 изображений ready,
p95=10.587ms,p99=17.222ms,456accepted spans, дополнительная потеря SDK→Collector0.
Результат — evidence/balanced-confirmation.json.
Сырые результаты: [observability-cost.json](evidence/observability-cost.json),
[cost-summary.json](evidence/cost-summary.json), [sampling-retention.json](evidence/sampling-retention.json).
![Цена наблюдаемости, 3 повтора](evidence/observability-cost.png)
## Проверки
Локально выполнено:
- make check и полная OpenAPI3.1 validation — PASS, оригинальные contracts/tests без изменений.
- make test и make test-student — public smoke и18тестов API/изображений PASS.
- make test-observability — реальная causal trace, retry, correlated logs, low-cardinality и canary checks PASS.
- make test-incidents — оба алерта firing/resolved, trace SQL lock и восстановление PASS.
- make test-telemetry-outage — бизнес-записи/worker при отказе Tempo, bounded memory, новые traces после возврата PASS.
- make test-persistence ACTION_SCRIPT=scripts/restart.sh — confirmed business data и старый Bearer сохранены;
дополнительный persistence_telemetry.py подтвердил прежние spans/logs и historical Prometheus sample.
- make test-failover ACTION_SCRIPT=scripts/stop-one.sh —120записей,1временная ошибка,
первый успех после отказа2.602s; API восстановлен.
- student-tests/security.py — anonymous object/list403, strict envelope PASS.
- student-tests/pipeline_checks.py —13групп инвариантов, реальные exit77, lease recovery/version fencing/replay/cleanup PASS.
- make research —12прогонов; отдельные sampling_retention.py —9синхронизированных прогонов.
- promtool check rules —3правила PASS.
- Grafana UI: dashboard с реальными рядами, Tempo→Loki по тому же trace_id и Loki TraceID→Tempo проверены.
Скриншоты evidence/grafana.jpg, trace-ui.jpg, correlated-ui.jpg.
Настоящий отдельный clone commit0e78b2b с новыми secrets и пустыми volumes:
сборка, миграции, public smoke,18тестов, causal/retry proof и полный restart с прежними
business data/Bearer/Tempo/Loki/Prometheus history — PASS за83.54s.
Тестовый Compose project mtusi-lab6-clean-6df218fa и только его volumes удалены;
основной mtusi-lab6 сохранён. Evidence: clean-clone.json / clean-clone.txt.
Изменения после clone ограничены отчётом, evidence и дополнительной проверкой исследования, бизнес-код не менялся.
Команды для повторения fault/research проверок запускать последовательно на отдельном учебном стенде:
они управляют только текущим Compose project и временно останавливают его компоненты.
CI application workflow добавлен; результат hosted CI не подтверждён, требующее maintainer approval выполнение не выдаётся за PASS.
Сохранённые txt/JSON — фактические локальные результаты. Raw telemetry содержит только синтетические технические ID.
## Ограничения
Один host, один PostgreSQL/S3/broker/LB/Collector/backend каждого типа остаются SPOF.
Доказывается отказ API-процесса, перезапуск с сохранением volumes и ограниченный отказ telemetry backend,
а не потеря диска/питания, distributed HA или exactly-once.
Миграции и outbox определяют корректность данных независимо от наличия telemetry.
Логи и метрики тоже имеют bounded asynchronous exporters; при перегрузке/отказе Collector возможны их потери.
Head sampling не гарантирует сохранение ошибки или редкого хвоста; для этого потребовалась бы отдельная политика tail sampling
с доказанным буфером/памятью, которую мы не заявляем реализованной.
Retention24h и file-based stores рассчитаны на локальный synthetic стенд, не промышленное хранение.
## Источники и использование ИИ
Собственная lab5, skeleton/contracts/public tests lab6.
Официальные материалы:
- https://opentelemetry.io/docs/languages/python/instrumentation/
- https://opentelemetry.io/docs/collector/configuration/
- https://opentelemetry.io/docs/concepts/context-propagation/
- https://grafana.com/docs/loki/latest/send-data/otel/
- https://grafana.com/docs/grafana/latest/administration/provisioning/
- https://prometheus.io/docs/practices/histograms/
Codex использован для реализации, проверки, controlled incidents, исследования и отчёта.
Численные результаты относятся к сохранённым локальным прогонам. Hosted CI не объявляется прошедшим без отдельного подтверждения.
\nНастройка Viewer/Explore: https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#viewers_can_edit\n
+298
View File
@@ -0,0 +1,298 @@
name: mtusi-lab6
x-application: &application
image: mtusi-messenger:lab6
networks: [backend]
environment: &environment
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
HTTP_WORKERS: ${HTTP_WORKERS:-16}
DB_POOL_MAX: ${DB_POOL_MAX:-8}
DB_ACCESS_MODE: ${DB_ACCESS_MODE:-optimized}
LOG_LEVEL: WARNING
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
TRACE_SAMPLE_RATIO: ${TRACE_SAMPLE_RATIO:-0.25}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
S3_ENDPOINT: http://s3:8333
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_key
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret
secrets: [database_url, cache_url, s3_api_key, s3_api_secret]
tmpfs: ["/tmp:size=128m,mode=1777"]
mem_limit: ${API_MEMORY:-512m}
cpus: ${API_CPUS:-1}
stop_grace_period: 10s
services:
migrate:
<<: *application
command: [python, src/migrate.py]
healthcheck:
disable: true
deploy:
replicas: 1
depends_on:
postgres:
condition: service_healthy
api:
<<: *application
build: .
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
otel-collector:
condition: service_started
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
bucket-bootstrap:
condition: service_completed_successfully
bucket-bootstrap:
<<: *application
command: [python, src/bucket_bootstrap.py]
environment:
<<: *environment
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret
secrets: [s3_admin_key, s3_admin_secret]
healthcheck:
disable: true
depends_on:
s3:
condition: service_healthy
audit:
<<: *application
profiles: [tools]
environment:
<<: *environment
BASE_URL: http://lb:8080
BROKER_URL_FILE: /run/secrets/broker_url
JOB_LEASE_SECONDS: 5
secrets: [database_url, cache_url, s3_api_key, s3_api_secret, s3_admin_key, s3_admin_secret, s3_worker_key, s3_worker_secret, broker_url]
healthcheck:
disable: true
deploy:
replicas: 1
dispatcher:
<<: *application
command: [python, src/pipeline.py, dispatch]
environment:
<<: *environment
BROKER_URL_FILE: /run/secrets/broker_url
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60}
secrets: [database_url, broker_url]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
restart: unless-stopped
worker:
<<: *application
command: [python, src/pipeline.py, work]
environment:
<<: *environment
BROKER_URL_FILE: /run/secrets/broker_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_key
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
secrets: [database_url, broker_url, s3_worker_key, s3_worker_secret]
mem_limit: ${WORKER_MEMORY:-768m}
tmpfs: ["/tmp:size=128m,mode=1777"]
cpus: 1
deploy:
replicas: 1
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
bucket-bootstrap:
condition: service_completed_successfully
broker:
condition: service_healthy
restart: unless-stopped
broker:
image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47
command: [redis-server, --appendonly, "yes", --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction, --save, ""]
volumes: [broker_data:/data]
networks: [backend]
mem_limit: 192m
cpus: 0.5
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 1s
retries: 30
s3-store:
image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8
command: [mini, -dir=/data, -ip=s3-store, -s3=false, -admin.ui=false, -webdav=false, -master.telemetry=false, -master.volumeSizeLimitMB=128, -volume.max=16, -volume.fileSizeLimitMB=11, -volume.concurrentUploadLimitMB=64, -volume.concurrentDownloadLimitMB=64]
networks: [storage]
volumes: [s3_data:/data]
mem_limit: 512m
cpus: 1
stop_grace_period: 20s
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8888/]
interval: 2s
timeout: 2s
retries: 45
s3:
image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8
command: [s3, -filer=s3-store:8888, -port=8333, -config=/run/secrets/s3_config, -iam=false, -port.iceberg=0, -port.lance=0, -concurrentFileUploadLimit=16, -concurrentUploadLimitMB=32, -readerCacheSizeMB=16]
secrets: [s3_config]
networks: [backend, storage]
mem_limit: 512m
environment:
GOMEMLIMIT: 384MiB
cpus: 0.5
depends_on:
s3-store:
condition: service_healthy
healthcheck:
test: [CMD-SHELL, 'nc -z 127.0.0.1 8333']
interval: 2s
timeout: 1s
retries: 30
lb:
image: haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949
ports: ["127.0.0.1:${APP_PORT:-18093}:8080"]
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
networks: [edge, backend]
depends_on:
api:
condition: service_healthy
mem_limit: 64m
cpus: 0.5
stop_grace_period: 12s
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 2s
timeout: 2s
retries: 10
postgres:
image: postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
networks: [backend]
healthcheck:
test: [CMD, pg_isready, -h, 127.0.0.1, -U, messenger, -d, messenger]
interval: 2s
timeout: 1s
retries: 30
mem_limit: 512m
cpus: ${POSTGRES_CPUS:-1}
redis:
image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47
command: [redis-server, --maxmemory, 32mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"]
networks: [backend]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 1s
retries: 30
mem_limit: 64m
cpus: 0.25
otel-collector:
image: otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af
command: [--config=/etc/otel/config.yaml]
volumes: [./infra/otel-collector.yaml:/etc/otel/config.yaml:ro]
networks: [backend, edge]
ports: ["127.0.0.1:${COLLECTOR_PORT:-18896}:8888"]
mem_limit: 256m
cpus: 0.5
environment:
GOMEMLIMIT: 192MiB
prometheus:
image: prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.path=/prometheus, --storage.tsdb.retention.time=24h, --storage.tsdb.retention.size=256MB]
volumes: [./infra/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro, ./infra/alerts.yaml:/etc/prometheus/alerts.yaml:ro, metrics_data:/prometheus]
networks: [backend, edge]
ports: ["127.0.0.1:${PROMETHEUS_PORT:-19096}:9090"]
mem_limit: 256m
cpus: 0.5
loki:
image: grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd
command: [-config.file=/etc/loki/config.yaml]
volumes: [./infra/loki.yaml:/etc/loki/config.yaml:ro, logs_data:/loki]
networks: [backend, edge]
ports: ["127.0.0.1:${LOKI_PORT:-13106}:3100"]
mem_limit: 256m
cpus: 0.5
environment:
GOMEMLIMIT: 192MiB
tempo:
image: grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7
command: [-config.file=/etc/tempo/config.yaml]
volumes: [./infra/tempo.yaml:/etc/tempo/config.yaml:ro, traces_data:/var/tempo]
networks: [backend, edge]
ports: ["127.0.0.1:${TEMPO_PORT:-13206}:3200"]
mem_limit: 512m
cpus: 0.5
environment:
GOMEMLIMIT: 384MiB
grafana:
image: grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d
volumes: [./infra/grafana/provisioning:/etc/grafana/provisioning:ro, ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro, grafana_data:/var/lib/grafana]
networks: [backend, edge]
ports: ["127.0.0.1:${GRAFANA_PORT:-13006}:3000"]
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
GF_AUTH_DISABLE_LOGIN_FORM: "true"
GF_AUTH_BASIC_ENABLED: "false"
GF_USERS_ALLOW_SIGN_UP: "false"
GF_USERS_VIEWERS_CAN_EDIT: "true"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
mem_limit: 256m
cpus: 0.5
volumes:
metrics_data:
logs_data:
traces_data:
grafana_data:
postgres_data:
broker_data:
s3_data:
networks:
edge:
storage:
internal: true
backend:
internal: true
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_admin_key:
file: ./secrets/s3_admin_key
s3_admin_secret:
file: ./secrets/s3_admin_secret
s3_api_key:
file: ./secrets/s3_api_key
s3_api_secret:
file: ./secrets/s3_api_secret
s3_config:
file: ./secrets/s3_config
broker_url:
file: ./secrets/broker_url
s3_worker_key:
file: ./secrets/s3_worker_key
s3_worker_secret:
file: ./secrets/s3_worker_secret
+100
View File
@@ -0,0 +1,100 @@
{
"duration_seconds": 56.09580111503601,
"confirmed_messages": 135,
"confirmed_attachments_ready": 5,
"all_confirmed_messages_retained": true,
"collector_samples": [
{
"elapsed": 2.1163909435272217,
"memory": "58.21MiB / 256MiB",
"cpu": "1.00%"
},
{
"elapsed": 14.031727075576782,
"memory": "58.21MiB / 256MiB",
"cpu": "0.59%"
},
{
"elapsed": 25.931473970413208,
"memory": "58.77MiB / 256MiB",
"cpu": "0.62%"
},
{
"elapsed": 37.78569412231445,
"memory": "62.01MiB / 256MiB",
"cpu": "0.69%"
},
{
"elapsed": 49.66645312309265,
"memory": "62.05MiB / 256MiB",
"cpu": "1.13%"
}
],
"before": {
"otelcol_exporter_queue_capacity": 256.0,
"otelcol_exporter_queue_size": 0.0,
"otelcol_exporter_send_failed_log_records_total": 0.0,
"otelcol_exporter_send_failed_metric_points_total": 0.0,
"otelcol_exporter_send_failed_spans_total": 0.0,
"otelcol_exporter_sent_log_records_total": 824.0,
"otelcol_exporter_sent_metric_points_total": 16842.0,
"otelcol_exporter_sent_spans_total": 1783.0,
"otelcol_process_cpu_seconds_total": 2.12,
"otelcol_process_memory_rss_bytes": 203309056.0,
"otelcol_process_runtime_heap_alloc_bytes": 32249488.0,
"otelcol_process_runtime_total_alloc_bytes_total": 333861280.0,
"otelcol_process_runtime_total_sys_memory_bytes": 62215432.0,
"otelcol_process_uptime_seconds_total": 401.657121304,
"otelcol_processor_accepted_log_records_total": 824.0,
"otelcol_processor_accepted_metric_points_total": 16917.0,
"otelcol_processor_accepted_spans_total": 1783.0,
"otelcol_processor_batch_batch_send_size_bucket": 10026.0,
"otelcol_processor_batch_batch_send_size_sum": 19449.0,
"otelcol_processor_batch_batch_send_size_count": 478.0,
"otelcol_processor_batch_batch_size_trigger_send_total": 11.0,
"otelcol_processor_batch_metadata_cardinality": 3.0,
"otelcol_processor_batch_timeout_trigger_send_total": 467.0,
"otelcol_processor_incoming_items_total": 19524.0,
"otelcol_processor_outgoing_items_total": 19524.0,
"otelcol_receiver_accepted_log_records_total": 824.0,
"otelcol_receiver_accepted_metric_points_total": 16917.0,
"otelcol_receiver_accepted_spans_total": 1783.0,
"otelcol_receiver_refused_log_records_total": 0.0,
"otelcol_receiver_refused_metric_points_total": 0.0,
"otelcol_receiver_refused_spans_total": 0.0
},
"during": {
"otelcol_exporter_queue_capacity": 256.0,
"otelcol_exporter_queue_size": 37.0,
"otelcol_exporter_send_failed_log_records_total": 0.0,
"otelcol_exporter_send_failed_metric_points_total": 0.0,
"otelcol_exporter_send_failed_spans_total": 156.0,
"otelcol_exporter_sent_log_records_total": 1021.0,
"otelcol_exporter_sent_metric_points_total": 19388.0,
"otelcol_exporter_sent_spans_total": 1801.0,
"otelcol_process_cpu_seconds_total": 2.54,
"otelcol_process_memory_rss_bytes": 206245888.0,
"otelcol_process_runtime_heap_alloc_bytes": 41796440.0,
"otelcol_process_runtime_total_alloc_bytes_total": 385908760.0,
"otelcol_process_runtime_total_sys_memory_bytes": 66409736.0,
"otelcol_process_uptime_seconds_total": 456.688764525,
"otelcol_processor_accepted_log_records_total": 1021.0,
"otelcol_processor_accepted_metric_points_total": 19388.0,
"otelcol_processor_accepted_spans_total": 2356.0,
"otelcol_processor_batch_batch_send_size_bucket": 12845.0,
"otelcol_processor_batch_batch_send_size_sum": 22765.0,
"otelcol_processor_batch_batch_send_size_count": 606.0,
"otelcol_processor_batch_batch_size_trigger_send_total": 11.0,
"otelcol_processor_batch_metadata_cardinality": 3.0,
"otelcol_processor_batch_timeout_trigger_send_total": 595.0,
"otelcol_processor_incoming_items_total": 22765.0,
"otelcol_processor_outgoing_items_total": 22765.0,
"otelcol_receiver_accepted_log_records_total": 1021.0,
"otelcol_receiver_accepted_metric_points_total": 19388.0,
"otelcol_receiver_accepted_spans_total": 2356.0,
"otelcol_receiver_refused_log_records_total": 0.0,
"otelcol_receiver_refused_metric_points_total": 0.0,
"otelcol_receiver_refused_spans_total": 0.0
},
"recovered_trace_id": "1555228135a04659bac5654623c7a552"
}
+141
View File
@@ -0,0 +1,141 @@
{
"mode": "balanced25",
"enabled": 1,
"sample_ratio": 0.25,
"repeat": 0,
"arrival_rps": 30,
"scheduled_requests": 600,
"confirmed_requests": 600,
"load_wall_seconds": 19.970493208,
"successful_rps": 30.044325583288316,
"p50_seconds": 0.003056625000001034,
"p95_seconds": 0.010586957999999314,
"p99_seconds": 0.017221916999998754,
"errors": [],
"image_jobs_ready": 30,
"resource": {
"api": {
"cpu_seconds": 2.6645160000000008,
"peak_sampled_anonymous_bytes": 197382144,
"peak_sampled_working_set_bytes": 206495744,
"rx_bytes": 4570530
},
"broker": {
"cpu_seconds": 0.15844099999999983,
"peak_sampled_anonymous_bytes": 3289088,
"peak_sampled_working_set_bytes": 3899392,
"rx_bytes": 46110
},
"dispatcher": {
"cpu_seconds": 0.7753760000000005,
"peak_sampled_anonymous_bytes": 45830144,
"peak_sampled_working_set_bytes": 48340992,
"rx_bytes": 472693
},
"grafana": {
"cpu_seconds": 0.34897599999999995,
"peak_sampled_anonymous_bytes": 106962944,
"peak_sampled_working_set_bytes": 114311168,
"rx_bytes": 77173
},
"lb": {
"cpu_seconds": 0.24454800000000043,
"peak_sampled_anonymous_bytes": 6942720,
"peak_sampled_working_set_bytes": 7999488,
"rx_bytes": 4287798
},
"loki": {
"cpu_seconds": 0.16779699999999975,
"peak_sampled_anonymous_bytes": 108310528,
"peak_sampled_working_set_bytes": 112246784,
"rx_bytes": 111200
},
"otel-collector": {
"cpu_seconds": 0.19291599999999987,
"peak_sampled_anonymous_bytes": 57331712,
"peak_sampled_working_set_bytes": 63811584,
"rx_bytes": 863090
},
"postgres": {
"cpu_seconds": 1.631062,
"peak_sampled_anonymous_bytes": 15544320,
"peak_sampled_working_set_bytes": 38952960,
"rx_bytes": 1139116
},
"prometheus": {
"cpu_seconds": 0.1952210000000003,
"peak_sampled_anonymous_bytes": 56045568,
"peak_sampled_working_set_bytes": 59535360,
"rx_bytes": 116869
},
"redis": {
"cpu_seconds": 0.17422799999999983,
"peak_sampled_anonymous_bytes": 3067904,
"peak_sampled_working_set_bytes": 3612672,
"rx_bytes": 119742
},
"s3": {
"cpu_seconds": 0.4350659999999995,
"peak_sampled_anonymous_bytes": 69742592,
"peak_sampled_working_set_bytes": 71454720,
"rx_bytes": 11144479
},
"s3-store": {
"cpu_seconds": 0.9894250000000007,
"peak_sampled_anonymous_bytes": 134537216,
"peak_sampled_working_set_bytes": 137109504,
"rx_bytes": 7486398
},
"tempo": {
"cpu_seconds": 0.11411999999999978,
"peak_sampled_anonymous_bytes": 360697856,
"peak_sampled_working_set_bytes": 364331008,
"rx_bytes": 37382
},
"worker": {
"cpu_seconds": 5.430313999999999,
"peak_sampled_anonymous_bytes": 77266944,
"peak_sampled_working_set_bytes": 82108416,
"rx_bytes": 3761105
}
},
"collector_ingest": {
"otelcol_receiver_accepted_spans_total": 456.0,
"otelcol_receiver_accepted_log_records_total": 840.0,
"otelcol_receiver_accepted_metric_points_total": 1154.0,
"otelcol_exporter_sent_spans_total": 456.0,
"otelcol_exporter_send_failed_spans_total": 0
},
"sampled_spans_finished": 456.0,
"sampled_spans_not_accepted_after_drain": 0,
"rare_probes": [
{
"request_id": "0c11495cc94443caadcd12bd17a86dd0",
"status": 503,
"latency_seconds": 1.4905514579999988,
"trace_sampled": true,
"trace_id": "aeacd5bdeb7d0f073b2a2cc44487c22c"
},
{
"request_id": "b0871c2ae52245ce882338cf01bd6a4d",
"status": 503,
"latency_seconds": 1.4968927090000008,
"trace_sampled": false,
"trace_id": "4d8c492fae68617ec5dbd2bc849de2e9"
},
{
"request_id": "d1d60c7ee15f498b988e031e15dc7d27",
"status": 503,
"latency_seconds": 1.4817318749999941,
"trace_sampled": false,
"trace_id": "931e742a91866246ac3b77dd062517c6"
},
{
"request_id": "95e79f3e1894491c9355c15e0e79a695",
"status": 200,
"latency_seconds": 1.1869852499999993,
"trace_sampled": false,
"trace_id": "34e1ad7e9c366b666caf53045be238e9"
}
]
}
+11
View File
@@ -0,0 +1,11 @@
{
"source_commit": "0e78b2b27de93464a6bd0974026bee4bfe4f602a",
"project": "mtusi-lab6-clean-6df218fa",
"new_secrets": true,
"new_empty_volumes": true,
"build_smoke_18_api_tests": true,
"real_observability_proof": true,
"business_and_telemetry_persistence": true,
"temporary_project_removed": true,
"seconds": 83.54
}
+338
View File
@@ -0,0 +1,338 @@
RUN python3 scripts/bootstrap.py
Local configuration ready; no secrets printed.
RUN docker compose build api
Image mtusi-messenger:lab6 Building
#1 [internal] load local bake definitions
#1 reading from stdin 707B done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 739B done
#2 DONE 0.0s
#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
#3 DONE 1.0s
#4 [internal] load .dockerignore
#4 transferring context: 203B done
#4 DONE 0.0s
#5 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
#5 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done
#5 DONE 0.0s
#6 [internal] load build context
#6 transferring context: 88.54kB done
#6 DONE 0.0s
#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
#7 CACHED
#8 [5/7] RUN pip install --no-cache-dir -r requirements.txt
#8 CACHED
#9 [6/7] COPY src/ ./src/
#9 CACHED
#10 [2/7] WORKDIR /app
#10 CACHED
#11 [4/7] COPY requirements.txt ./
#11 CACHED
#12 [7/7] COPY migrations/ ./migrations/
#12 CACHED
#13 exporting to image
#13 exporting layers done
#13 exporting manifest sha256:e533c95082fcc2d03df488d24c04c74c3205f96ebce7d39fcf544e15ed2ac2c9 done
#13 exporting config sha256:d5159a690cc9eed35b1e1df5510f9f8595fbb5e303c09bde83ea92197bbfab8c done
#13 exporting attestation manifest sha256:91fc899a62ccb18479e1135cbfe8c85ae865353f7315c7c8897a6418fe283c11 done
#13 exporting manifest list sha256:a0c792a12285d3f4914e1ce37a8832777d639abd2653a71ff0b581b28f2f5e6b done
#13 naming to docker.io/library/mtusi-messenger:lab6 done
#13 unpacking to docker.io/library/mtusi-messenger:lab6 done
#13 DONE 0.0s
#14 resolving provenance for metadata file
#14 DONE 0.0s
Image mtusi-messenger:lab6 Built
RUN docker compose up -d --wait --wait-timeout 180
Network mtusi-lab6-clean-6df218fa_edge Creating
Network mtusi-lab6-clean-6df218fa_edge Created
Network mtusi-lab6-clean-6df218fa_backend Creating
Network mtusi-lab6-clean-6df218fa_backend Created
Network mtusi-lab6-clean-6df218fa_storage Creating
Network mtusi-lab6-clean-6df218fa_storage Created
Volume mtusi-lab6-clean-6df218fa_logs_data Creating
Volume mtusi-lab6-clean-6df218fa_logs_data Created
Volume mtusi-lab6-clean-6df218fa_metrics_data Creating
Volume mtusi-lab6-clean-6df218fa_metrics_data Created
Volume mtusi-lab6-clean-6df218fa_traces_data Creating
Volume mtusi-lab6-clean-6df218fa_traces_data Created
Volume mtusi-lab6-clean-6df218fa_postgres_data Creating
Volume mtusi-lab6-clean-6df218fa_postgres_data Created
Volume mtusi-lab6-clean-6df218fa_s3_data Creating
Volume mtusi-lab6-clean-6df218fa_s3_data Created
Volume mtusi-lab6-clean-6df218fa_broker_data Creating
Volume mtusi-lab6-clean-6df218fa_broker_data Created
Volume mtusi-lab6-clean-6df218fa_grafana_data Creating
Volume mtusi-lab6-clean-6df218fa_grafana_data Created
Container mtusi-lab6-clean-6df218fa-grafana-1 Creating
Container mtusi-lab6-clean-6df218fa-s3-store-1 Creating
Container mtusi-lab6-clean-6df218fa-prometheus-1 Creating
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Creating
Container mtusi-lab6-clean-6df218fa-loki-1 Creating
Container mtusi-lab6-clean-6df218fa-redis-1 Creating
Container mtusi-lab6-clean-6df218fa-postgres-1 Creating
Container mtusi-lab6-clean-6df218fa-broker-1 Creating
Container mtusi-lab6-clean-6df218fa-tempo-1 Creating
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Created
Container mtusi-lab6-clean-6df218fa-grafana-1 Created
Container mtusi-lab6-clean-6df218fa-prometheus-1 Created
Container mtusi-lab6-clean-6df218fa-loki-1 Created
Container mtusi-lab6-clean-6df218fa-tempo-1 Created
Container mtusi-lab6-clean-6df218fa-s3-store-1 Created
Container mtusi-lab6-clean-6df218fa-broker-1 Created
Container mtusi-lab6-clean-6df218fa-s3-1 Creating
Container mtusi-lab6-clean-6df218fa-redis-1 Created
Container mtusi-lab6-clean-6df218fa-postgres-1 Created
Container mtusi-lab6-clean-6df218fa-migrate-1 Creating
Container mtusi-lab6-clean-6df218fa-migrate-1 Created
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Creating
Container mtusi-lab6-clean-6df218fa-s3-1 Created
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Creating
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Created
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Created
Container mtusi-lab6-clean-6df218fa-worker-1 Creating
Container mtusi-lab6-clean-6df218fa-api-1 Creating
Container mtusi-lab6-clean-6df218fa-api-2 Creating
Container mtusi-lab6-clean-6df218fa-api-1 Created
Container mtusi-lab6-clean-6df218fa-api-2 Created
Container mtusi-lab6-clean-6df218fa-lb-1 Creating
Container mtusi-lab6-clean-6df218fa-worker-1 Created
Container mtusi-lab6-clean-6df218fa-lb-1 Created
Container mtusi-lab6-clean-6df218fa-s3-store-1 Starting
Container mtusi-lab6-clean-6df218fa-redis-1 Starting
Container mtusi-lab6-clean-6df218fa-tempo-1 Starting
Container mtusi-lab6-clean-6df218fa-prometheus-1 Starting
Container mtusi-lab6-clean-6df218fa-loki-1 Starting
Container mtusi-lab6-clean-6df218fa-broker-1 Starting
Container mtusi-lab6-clean-6df218fa-grafana-1 Starting
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Starting
Container mtusi-lab6-clean-6df218fa-postgres-1 Starting
Container mtusi-lab6-clean-6df218fa-s3-store-1 Started
Container mtusi-lab6-clean-6df218fa-s3-store-1 Waiting
Container mtusi-lab6-clean-6df218fa-broker-1 Started
Container mtusi-lab6-clean-6df218fa-redis-1 Started
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Started
Container mtusi-lab6-clean-6df218fa-prometheus-1 Started
Container mtusi-lab6-clean-6df218fa-loki-1 Started
Container mtusi-lab6-clean-6df218fa-tempo-1 Started
Container mtusi-lab6-clean-6df218fa-postgres-1 Started
Container mtusi-lab6-clean-6df218fa-postgres-1 Waiting
Container mtusi-lab6-clean-6df218fa-grafana-1 Started
Container mtusi-lab6-clean-6df218fa-postgres-1 Healthy
Container mtusi-lab6-clean-6df218fa-migrate-1 Starting
Container mtusi-lab6-clean-6df218fa-migrate-1 Started
Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
Container mtusi-lab6-clean-6df218fa-broker-1 Waiting
Container mtusi-lab6-clean-6df218fa-broker-1 Healthy
Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Starting
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Started
Container mtusi-lab6-clean-6df218fa-s3-store-1 Healthy
Container mtusi-lab6-clean-6df218fa-s3-1 Starting
Container mtusi-lab6-clean-6df218fa-s3-1 Started
Container mtusi-lab6-clean-6df218fa-s3-1 Waiting
Container mtusi-lab6-clean-6df218fa-s3-1 Healthy
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Starting
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Started
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting
Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
Container mtusi-lab6-clean-6df218fa-redis-1 Waiting
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting
Container mtusi-lab6-clean-6df218fa-broker-1 Waiting
Container mtusi-lab6-clean-6df218fa-broker-1 Healthy
Container mtusi-lab6-clean-6df218fa-redis-1 Healthy
Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited
Container mtusi-lab6-clean-6df218fa-worker-1 Starting
Container mtusi-lab6-clean-6df218fa-api-1 Starting
Container mtusi-lab6-clean-6df218fa-worker-1 Started
Container mtusi-lab6-clean-6df218fa-api-1 Started
Container mtusi-lab6-clean-6df218fa-api-2 Starting
Container mtusi-lab6-clean-6df218fa-api-2 Started
Container mtusi-lab6-clean-6df218fa-api-1 Waiting
Container mtusi-lab6-clean-6df218fa-api-2 Waiting
Container mtusi-lab6-clean-6df218fa-api-1 Healthy
Container mtusi-lab6-clean-6df218fa-api-2 Healthy
Container mtusi-lab6-clean-6df218fa-lb-1 Starting
Container mtusi-lab6-clean-6df218fa-lb-1 Started
Container mtusi-lab6-clean-6df218fa-broker-1 Waiting
Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
Container mtusi-lab6-clean-6df218fa-worker-1 Waiting
Container mtusi-lab6-clean-6df218fa-prometheus-1 Waiting
Container mtusi-lab6-clean-6df218fa-lb-1 Waiting
Container mtusi-lab6-clean-6df218fa-loki-1 Waiting
Container mtusi-lab6-clean-6df218fa-tempo-1 Waiting
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Waiting
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Waiting
Container mtusi-lab6-clean-6df218fa-grafana-1 Waiting
Container mtusi-lab6-clean-6df218fa-redis-1 Waiting
Container mtusi-lab6-clean-6df218fa-s3-store-1 Waiting
Container mtusi-lab6-clean-6df218fa-postgres-1 Waiting
Container mtusi-lab6-clean-6df218fa-s3-1 Waiting
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting
Container mtusi-lab6-clean-6df218fa-api-1 Waiting
Container mtusi-lab6-clean-6df218fa-api-2 Waiting
Container mtusi-lab6-clean-6df218fa-s3-store-1 Healthy
Container mtusi-lab6-clean-6df218fa-loki-1 Healthy
Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
Container mtusi-lab6-clean-6df218fa-broker-1 Healthy
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited
Container mtusi-lab6-clean-6df218fa-postgres-1 Healthy
Container mtusi-lab6-clean-6df218fa-worker-1 Healthy
Container mtusi-lab6-clean-6df218fa-tempo-1 Healthy
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Healthy
Container mtusi-lab6-clean-6df218fa-grafana-1 Healthy
Container mtusi-lab6-clean-6df218fa-redis-1 Healthy
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Healthy
Container mtusi-lab6-clean-6df218fa-s3-1 Healthy
Container mtusi-lab6-clean-6df218fa-prometheus-1 Healthy
Container mtusi-lab6-clean-6df218fa-api-1 Healthy
Container mtusi-lab6-clean-6df218fa-api-2 Healthy
Container mtusi-lab6-clean-6df218fa-lb-1 Healthy
RUN python3 scripts/wait_ready.py
Ready through load balancer.
RUN make check test test-student BASE_URL=http://127.0.0.1:18094
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
python3 tests/smoke.py --base-url "http://127.0.0.1:18094"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
BASE_URL="http://127.0.0.1:18094" python3 -m unittest discover -s student-tests -p 'test_*.py' -v
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
test_cursor_scope_and_validation (test_api.APIContract) ... ok
test_foreign_chat (test_api.APIContract) ... ok
test_idempotency_concurrent_replicas (test_api.APIContract) ... ok
test_idempotency_conflicting_race (test_api.APIContract) ... ok
test_idempotency_invalid_keys (test_api.APIContract) ... ok
test_idempotency_scope (test_api.APIContract) ... ok
test_idempotency_semantic_body (test_api.APIContract) ... ok
test_invalid_chat_creation (test_api.APIContract) ... ok
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
test_protocol_error_headers (test_api.APIContract) ... ok
test_registration_race (test_api.APIContract) ... ok
test_unicode_limits (test_api.APIContract) ... ok
test_unicode_nul_round_trip (test_api.APIContract) ... ok
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
test_async_variants_and_replay (test_images.Images) ... ok
test_mime_bytes_pixels_and_filename (test_images.Images) ... ok
test_private_and_cross_chat (test_images.Images) ... ok
----------------------------------------------------------------------
Ran 18 tests in 4.950s
OK
RUN python3 -u student-tests/observability.py proof
Container mtusi-lab6-clean-6df218fa-worker-1 Stopping
Container mtusi-lab6-clean-6df218fa-worker-1 Stopped
Container mtusi-lab6-clean-6df218fa-s3-1 Paused
Container mtusi-lab6-clean-6df218fa-worker-1 Starting
Container mtusi-lab6-clean-6df218fa-worker-1 Started
Container mtusi-lab6-clean-6df218fa-s3-1 Unpaused
Container mtusi-lab6-clean-6df218fa-worker-1 Running
PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.
RUN python3 -u student-tests/persistence_telemetry.py
PASS: full business restart, original Bearer, historical Tempo trace, Loki logs and fixed-time Prometheus sample.
RUN docker compose down -v --remove-orphans
Container mtusi-lab6-clean-6df218fa-loki-1 Stopping
Container mtusi-lab6-clean-6df218fa-lb-1 Stopping
Container mtusi-lab6-clean-6df218fa-grafana-1 Stopping
Container mtusi-lab6-clean-6df218fa-tempo-1 Stopping
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Stopping
Container mtusi-lab6-clean-6df218fa-prometheus-1 Stopping
Container mtusi-lab6-clean-6df218fa-worker-1 Stopping
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Stopped
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Removing
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Removed
Container mtusi-lab6-clean-6df218fa-lb-1 Stopped
Container mtusi-lab6-clean-6df218fa-lb-1 Removing
Container mtusi-lab6-clean-6df218fa-lb-1 Removed
Container mtusi-lab6-clean-6df218fa-api-2 Stopping
Container mtusi-lab6-clean-6df218fa-api-1 Stopping
Container mtusi-lab6-clean-6df218fa-worker-1 Stopped
Container mtusi-lab6-clean-6df218fa-worker-1 Removing
Container mtusi-lab6-clean-6df218fa-worker-1 Removed
Container mtusi-lab6-clean-6df218fa-broker-1 Stopping
Container mtusi-lab6-clean-6df218fa-grafana-1 Stopped
Container mtusi-lab6-clean-6df218fa-grafana-1 Removing
Container mtusi-lab6-clean-6df218fa-prometheus-1 Stopped
Container mtusi-lab6-clean-6df218fa-prometheus-1 Removing
Container mtusi-lab6-clean-6df218fa-grafana-1 Removed
Container mtusi-lab6-clean-6df218fa-prometheus-1 Removed
Container mtusi-lab6-clean-6df218fa-broker-1 Stopped
Container mtusi-lab6-clean-6df218fa-broker-1 Removing
Container mtusi-lab6-clean-6df218fa-broker-1 Removed
Container mtusi-lab6-clean-6df218fa-api-1 Stopped
Container mtusi-lab6-clean-6df218fa-api-1 Removing
Container mtusi-lab6-clean-6df218fa-api-1 Removed
Container mtusi-lab6-clean-6df218fa-api-2 Stopped
Container mtusi-lab6-clean-6df218fa-api-2 Removing
Container mtusi-lab6-clean-6df218fa-api-2 Removed
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Stopping
Container mtusi-lab6-clean-6df218fa-migrate-1 Stopping
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Stopping
Container mtusi-lab6-clean-6df218fa-redis-1 Stopping
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Stopped
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Removing
Container mtusi-lab6-clean-6df218fa-migrate-1 Stopped
Container mtusi-lab6-clean-6df218fa-migrate-1 Removing
Container mtusi-lab6-clean-6df218fa-migrate-1 Removed
Container mtusi-lab6-clean-6df218fa-postgres-1 Stopping
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Removed
Container mtusi-lab6-clean-6df218fa-s3-1 Stopping
Container mtusi-lab6-clean-6df218fa-redis-1 Stopped
Container mtusi-lab6-clean-6df218fa-redis-1 Removing
Container mtusi-lab6-clean-6df218fa-postgres-1 Stopped
Container mtusi-lab6-clean-6df218fa-postgres-1 Removing
Container mtusi-lab6-clean-6df218fa-redis-1 Removed
Container mtusi-lab6-clean-6df218fa-postgres-1 Removed
Container mtusi-lab6-clean-6df218fa-loki-1 Stopped
Container mtusi-lab6-clean-6df218fa-loki-1 Removing
Container mtusi-lab6-clean-6df218fa-loki-1 Removed
Container mtusi-lab6-clean-6df218fa-s3-1 Stopped
Container mtusi-lab6-clean-6df218fa-s3-1 Removing
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Stopped
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Removing
Container mtusi-lab6-clean-6df218fa-s3-1 Removed
Container mtusi-lab6-clean-6df218fa-s3-store-1 Stopping
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Removed
Container mtusi-lab6-clean-6df218fa-s3-store-1 Stopped
Container mtusi-lab6-clean-6df218fa-s3-store-1 Removing
Container mtusi-lab6-clean-6df218fa-s3-store-1 Removed
Container mtusi-lab6-clean-6df218fa-tempo-1 Stopped
Container mtusi-lab6-clean-6df218fa-tempo-1 Removing
Container mtusi-lab6-clean-6df218fa-tempo-1 Removed
Volume mtusi-lab6-clean-6df218fa_broker_data Removing
Network mtusi-lab6-clean-6df218fa_storage Removing
Volume mtusi-lab6-clean-6df218fa_metrics_data Removing
Volume mtusi-lab6-clean-6df218fa_grafana_data Removing
Volume mtusi-lab6-clean-6df218fa_traces_data Removing
Network mtusi-lab6-clean-6df218fa_edge Removing
Volume mtusi-lab6-clean-6df218fa_s3_data Removing
Volume mtusi-lab6-clean-6df218fa_logs_data Removing
Volume mtusi-lab6-clean-6df218fa_postgres_data Removing
Network mtusi-lab6-clean-6df218fa_backend Removing
Volume mtusi-lab6-clean-6df218fa_broker_data Removed
Volume mtusi-lab6-clean-6df218fa_metrics_data Removed
Volume mtusi-lab6-clean-6df218fa_logs_data Removed
Volume mtusi-lab6-clean-6df218fa_grafana_data Removed
Volume mtusi-lab6-clean-6df218fa_traces_data Removed
Volume mtusi-lab6-clean-6df218fa_s3_data Removed
Volume mtusi-lab6-clean-6df218fa_postgres_data Removed
Network mtusi-lab6-clean-6df218fa_storage Removed
Network mtusi-lab6-clean-6df218fa_backend Removed
Network mtusi-lab6-clean-6df218fa_edge Removed
Binary file not shown.

After

Width:  |  Height:  |  Size: 79 KiB

+86
View File
@@ -0,0 +1,86 @@
{
"off": {
"runs": 3,
"requests": 1800,
"ready_images": 90,
"p95_ms": 9.81791700002077,
"p99_ms": 15.300541999977213,
"api_cpu_seconds": 2.3706709999999998,
"worker_cpu_seconds": 5.463901,
"collector_cpu_seconds": 0.053601000000000454,
"api_sum_sampled_anon_peak_mib": 159.59765625,
"collector_anon_peak_mib": 56.015625,
"accepted_spans": 0,
"accepted_logs": 0,
"accepted_metric_points": 0,
"collector_receive_kib": 13.2080078125,
"sampled_spans_not_accepted": 0,
"error_probe_traces_kept": 0,
"error_probes": 1,
"slow_probe_traces_kept": 0,
"slow_probes": 11
},
"full": {
"runs": 3,
"requests": 1800,
"ready_images": 90,
"p95_ms": 9.695250000049782,
"p99_ms": 14.552875000049426,
"api_cpu_seconds": 2.7994320000000004,
"worker_cpu_seconds": 5.533063,
"collector_cpu_seconds": 0.19410300000000014,
"api_sum_sampled_anon_peak_mib": 147.90234375,
"collector_anon_peak_mib": 56.02734375,
"accepted_spans": 1950.0,
"accepted_logs": 840.0,
"accepted_metric_points": 1011.0,
"collector_receive_kib": 1067.025390625,
"sampled_spans_not_accepted": 0,
"error_probe_traces_kept": 4,
"error_probes": 4,
"slow_probe_traces_kept": 8,
"slow_probes": 8
},
"sample10": {
"runs": 3,
"requests": 1800,
"ready_images": 90,
"p95_ms": 9.149458999957005,
"p99_ms": 13.731374999991885,
"api_cpu_seconds": 2.540737,
"worker_cpu_seconds": 5.2699810000000005,
"collector_cpu_seconds": 0.16662400000000055,
"api_sum_sampled_anon_peak_mib": 168.65625,
"collector_anon_peak_mib": 58.78125,
"accepted_spans": 176.0,
"accepted_logs": 840.0,
"accepted_metric_points": 987.0,
"collector_receive_kib": 720.869140625,
"sampled_spans_not_accepted": 0,
"error_probe_traces_kept": 0,
"error_probes": 1,
"slow_probe_traces_kept": 0,
"slow_probes": 11
},
"sample25": {
"runs": 3,
"requests": 1800,
"ready_images": 90,
"p95_ms": 10.556667000003017,
"p99_ms": 15.699083000015435,
"api_cpu_seconds": 2.6178429999999997,
"worker_cpu_seconds": 5.2202720000000005,
"collector_cpu_seconds": 0.18281499999999973,
"api_sum_sampled_anon_peak_mib": 147.81640625,
"collector_anon_peak_mib": 58.71875,
"accepted_spans": 467.0,
"accepted_logs": 840.0,
"accepted_metric_points": 1011.0,
"collector_receive_kib": 802.8349609375,
"sampled_spans_not_accepted": 0,
"error_probe_traces_kept": 1,
"error_probes": 2,
"slow_probe_traces_kept": 3,
"slow_probes": 10
}
}
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18093"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 1, "first_recovery_seconds": 2.602}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+28
View File
@@ -0,0 +1,28 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
python3 tests/smoke.py --base-url "http://127.0.0.1:18093"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
BASE_URL="http://127.0.0.1:18093" python3 -m unittest discover -s student-tests -p 'test_*.py' -v
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
test_cursor_scope_and_validation (test_api.APIContract) ... ok
test_foreign_chat (test_api.APIContract) ... ok
test_idempotency_concurrent_replicas (test_api.APIContract) ... ok
test_idempotency_conflicting_race (test_api.APIContract) ... ok
test_idempotency_invalid_keys (test_api.APIContract) ... ok
test_idempotency_scope (test_api.APIContract) ... ok
test_idempotency_semantic_body (test_api.APIContract) ... ok
test_invalid_chat_creation (test_api.APIContract) ... ok
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
test_protocol_error_headers (test_api.APIContract) ... ok
test_registration_race (test_api.APIContract) ... ok
test_unicode_limits (test_api.APIContract) ... ok
test_unicode_nul_round_trip (test_api.APIContract) ... ok
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
test_async_variants_and_replay (test_images.Images) ... ok
test_mime_bytes_pixels_and_filename (test_images.Images) ... ok
test_private_and_cross_chat (test_images.Images) ... ok
----------------------------------------------------------------------
Ran 18 tests in 5.414s
OK
+1
View File
@@ -0,0 +1 @@
PASS: Python syntax; original README/COURSE/CONTRIBUTING/contracts/public tests unchanged; actual local PostgreSQL password and S3 credentials absent from tracked/new files. Non-secret internal Redis service URLs intentionally appear in bootstrap.
Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 165 KiB

File diff suppressed because it is too large Load Diff

After

Width:  |  Height:  |  Size: 145 KiB

+8
View File
@@ -0,0 +1,8 @@
Container mtusi-lab6-worker-1 Stopping
Container mtusi-lab6-worker-1 Stopped
Container mtusi-lab6-s3-1 Paused
Container mtusi-lab6-worker-1 Starting
Container mtusi-lab6-worker-1 Started
Container mtusi-lab6-s3-1 Unpaused
Container mtusi-lab6-worker-1 Running
PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18093"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+119
View File
@@ -0,0 +1,119 @@
{
"invariants": {
"commit_before_publish_recovered": true,
"publish_before_marker_duplicate_safe": true,
"crashes_before_after_side_effect_and_commit": true,
"invalid_envelope_observable_dlq": true,
"duplicate_and_reorder_no_second_effect": true,
"concurrent_duplicate_single_claim": true,
"stale_running_version_fenced": true,
"old_delivery_after_new_ready_no_overwrite": true,
"failed_registry_explicit_replay_and_ready_refusal": true,
"transient_backoff_exhaustion_then_replay": true,
"invalid_decode_permanent_failed": true,
"cleanup_preserves_current_and_fresh_failed": true,
"ready_staging_cleanup_idempotent": true
},
"faults": [
{
"point": "before_side_effect",
"exit_code": 77,
"state_at_crash": "processing",
"recovered": "ready",
"attempts": 2,
"result_objects": 4
},
{
"point": "after_original",
"exit_code": 77,
"state_at_crash": "processing",
"recovered": "ready",
"attempts": 2,
"result_objects": 4
},
{
"point": "after_objects",
"exit_code": 77,
"state_at_crash": "processing",
"recovered": "ready",
"attempts": 2,
"result_objects": 4
},
{
"point": "after_commit",
"exit_code": 77,
"state_at_crash": "ready",
"recovered": "ready",
"attempts": 1,
"result_objects": 4
}
],
"retry": {
"transitions": [
{
"state": "queued",
"attempt": 1,
"error_code": "storage_unavailable"
},
{
"state": "failed",
"attempt": 2,
"error_code": "retry_exhausted"
}
],
"seconds": 1.983,
"replayed_attempt": 3
},
"transforms": [
{
"kind": "alpha",
"size": [
20,
10
],
"exif_removed": true
},
{
"kind": "exif",
"size": [
256,
160
],
"exif_removed": true
},
{
"kind": "small",
"size": [
5,
3
],
"exif_removed": true
}
],
"cleanup": [
{
"apply": false,
"retention_seconds": 86400,
"counts": {
"ready_staging": 397,
"expired_staging": 0
}
},
{
"apply": true,
"retention_seconds": 86400,
"counts": {
"ready_staging": 397,
"expired_staging": 0
}
},
{
"apply": true,
"retention_seconds": 86400,
"counts": {
"ready_staging": 0,
"expired_staging": 0
}
}
]
}
+546
View File
@@ -0,0 +1,546 @@
[
{
"container": "cf2cfedf29de75b2458ec532a42f264cdb7ef71435900a7ef1c478d9d6b2e61f",
"name": "/mtusi-lab6-api-1",
"service": "api",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:45:44.122480904Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
}
]
},
{
"container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b",
"name": "/mtusi-lab6-api-2",
"service": "api",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:45:44.196103962Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
}
]
},
{
"container": "13d6ed83fc36b28aa57fc7cf302d9624098934b1f9c6ec68254a2b1a5f95228d",
"name": "/mtusi-lab6-broker-1",
"service": "broker",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-11-20T08:45:30.515512908Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 201326592,
"nano_cpus": 500000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab6_broker_data"
}
]
},
{
"container": "076e71ce04ff5adbc19738e63a8ed319ad6f645183e6e65e47aec7dd97afe926",
"name": "/mtusi-lab6-bucket-bootstrap-1",
"service": "bucket-bootstrap",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": false,
"started_at": "2026-11-20T08:45:43.045813539Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_admin_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_admin_secret",
"name": null
}
]
},
{
"container": "b2de6e448b4102ff9f25833bf99cd5a9978947dae0c3e28e743c3ef18f58b943",
"name": "/mtusi-lab6-dispatcher-1",
"service": "dispatcher",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:45:43.627780758Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
}
]
},
{
"container": "54edeaec98aff44bfe34f8c103c33694187550a9b690cde7648017ad323d7310",
"name": "/mtusi-lab6-grafana-1",
"service": "grafana",
"image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d",
"user": "472",
"running": true,
"started_at": "2026-11-20T08:45:30.734589337Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 500000000,
"ports": {
"3000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "13006"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/etc/grafana/provisioning",
"name": null
},
{
"type": "bind",
"destination": "/var/lib/grafana/dashboards",
"name": null
},
{
"type": "volume",
"destination": "/var/lib/grafana",
"name": "mtusi-lab6_grafana_data"
}
]
},
{
"container": "7db8217af00aa7be6a7e3d19f656faf329598ef85bfc8fb2fc2d15ec0cf72c28",
"name": "/mtusi-lab6-lb-1",
"service": "lb",
"image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
"user": "haproxy",
"running": true,
"started_at": "2026-11-20T08:45:49.83573792Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 500000000,
"ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "18093"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/usr/local/etc/haproxy/haproxy.cfg",
"name": null
}
]
},
{
"container": "712535508b37fe699452fb2a4ed62a005a8cfa7a14e2b9827a9c82cd4ec982c9",
"name": "/mtusi-lab6-loki-1",
"service": "loki",
"image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd",
"user": "10001",
"running": true,
"started_at": "2026-11-20T08:45:31.075549983Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 500000000,
"ports": {
"3100/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "13106"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/etc/loki/config.yaml",
"name": null
},
{
"type": "volume",
"destination": "/loki",
"name": "mtusi-lab6_logs_data"
}
]
},
{
"container": "6374cc6d3e2b163264c18e1181ea5be5b53f365a5324686a908ea8f94861a6c7",
"name": "/mtusi-lab6-migrate-1",
"service": "migrate",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": false,
"started_at": "2026-11-20T08:45:42.539353205Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
}
]
},
{
"container": "89012420d895c4c7c0d9fdaf7c1ff40c3a5f97dca279a0ef8cec0be3903dce42",
"name": "/mtusi-lab6-otel-collector-1",
"service": "otel-collector",
"image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:45:30.652417444Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 500000000,
"ports": {
"4317/tcp": null,
"4318/tcp": null,
"55679/tcp": null,
"8888/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "18896"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/etc/otel/config.yaml",
"name": null
}
]
},
{
"container": "2257079aebc7a27eda5607608d7c4e7df3d1a196eb2c65d7236a1b60f7466c91",
"name": "/mtusi-lab6-postgres-1",
"service": "postgres",
"image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
"user": "",
"running": true,
"started_at": "2026-11-20T08:45:30.708099365Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"5432/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/var/lib/postgresql/data",
"name": "mtusi-lab6_postgres_data"
},
{
"type": "bind",
"destination": "/run/secrets/postgres_password",
"name": null
}
]
},
{
"container": "937df26581a582233f77674d7c66a0b7fbba5fffd2377977ebd13857cd510a66",
"name": "/mtusi-lab6-prometheus-1",
"service": "prometheus",
"image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996",
"user": "nobody",
"running": true,
"started_at": "2026-11-20T08:45:30.770516143Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 500000000,
"ports": {
"9090/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "19096"
}
]
},
"mounts": [
{
"type": "volume",
"destination": "/prometheus",
"name": "mtusi-lab6_metrics_data"
},
{
"type": "bind",
"destination": "/etc/prometheus/prometheus.yaml",
"name": null
},
{
"type": "bind",
"destination": "/etc/prometheus/alerts.yaml",
"name": null
}
]
},
{
"container": "cc255a9bdb8967c853de2a4f167fe269a1a4f004e0b88ab174c79397212fd2a4",
"name": "/mtusi-lab6-redis-1",
"service": "redis",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-11-20T08:45:30.676898647Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 250000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "db7892dbab7e8fc10b9d4431ee816406d604a540d7068b70aef208767168a178"
}
]
},
{
"container": "c4564618c7c266a67ed1f6a05989f8156ca42582a1860fe8d868fc758d18d4f3",
"name": "/mtusi-lab6-s3-1",
"service": "s3",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-11-20T08:45:30.688051331Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 500000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_config",
"name": null
},
{
"type": "volume",
"destination": "/data",
"name": "3f039af87109799a85e80069eb418dd6c13b9b036ffc46fb0e6ed1bfc21407de"
}
]
},
{
"container": "d3abacb0bdaf348f062e5ec77599ea8a0450872a17b450b190ba7bce78166129",
"name": "/mtusi-lab6-s3-store-1",
"service": "s3-store",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-11-20T08:45:31.797812564Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab6_s3_data"
}
]
},
{
"container": "9cfc7a5053725737ecfc90211eed31a0c97a15aeb63ca270b9596c4af69de808",
"name": "/mtusi-lab6-tempo-1",
"service": "tempo",
"image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:45:40.551529375Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 500000000,
"ports": {
"3200/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "13206"
}
]
},
"mounts": [
{
"type": "volume",
"destination": "/var/tempo",
"name": "mtusi-lab6_traces_data"
},
{
"type": "bind",
"destination": "/etc/tempo/config.yaml",
"name": null
}
]
},
{
"container": "ef14a69e994f9f18fdb3438a03a28a974a3fcfc564f2d0eedeff3949a8f9bb29",
"name": "/mtusi-lab6-worker-1",
"service": "worker",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:45:44.122215389Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 805306368,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_secret",
"name": null
}
]
}
]
+546
View File
@@ -0,0 +1,546 @@
[
{
"container": "cf2cfedf29de75b2458ec532a42f264cdb7ef71435900a7ef1c478d9d6b2e61f",
"name": "/mtusi-lab6-api-1",
"service": "api",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:43:28.014761078Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
}
]
},
{
"container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b",
"name": "/mtusi-lab6-api-2",
"service": "api",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:43:28.105739751Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
}
]
},
{
"container": "13d6ed83fc36b28aa57fc7cf302d9624098934b1f9c6ec68254a2b1a5f95228d",
"name": "/mtusi-lab6-broker-1",
"service": "broker",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-11-20T08:20:26.890287569Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 201326592,
"nano_cpus": 500000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab6_broker_data"
}
]
},
{
"container": "076e71ce04ff5adbc19738e63a8ed319ad6f645183e6e65e47aec7dd97afe926",
"name": "/mtusi-lab6-bucket-bootstrap-1",
"service": "bucket-bootstrap",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": false,
"started_at": "2026-11-20T08:40:43.538584225Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_admin_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_admin_secret",
"name": null
}
]
},
{
"container": "b2de6e448b4102ff9f25833bf99cd5a9978947dae0c3e28e743c3ef18f58b943",
"name": "/mtusi-lab6-dispatcher-1",
"service": "dispatcher",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:43:28.015064203Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
}
]
},
{
"container": "54edeaec98aff44bfe34f8c103c33694187550a9b690cde7648017ad323d7310",
"name": "/mtusi-lab6-grafana-1",
"service": "grafana",
"image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d",
"user": "472",
"running": true,
"started_at": "2026-11-20T08:21:11.514848908Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 500000000,
"ports": {
"3000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "13006"
}
]
},
"mounts": [
{
"type": "volume",
"destination": "/var/lib/grafana",
"name": "mtusi-lab6_grafana_data"
},
{
"type": "bind",
"destination": "/etc/grafana/provisioning",
"name": null
},
{
"type": "bind",
"destination": "/var/lib/grafana/dashboards",
"name": null
}
]
},
{
"container": "7db8217af00aa7be6a7e3d19f656faf329598ef85bfc8fb2fc2d15ec0cf72c28",
"name": "/mtusi-lab6-lb-1",
"service": "lb",
"image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
"user": "haproxy",
"running": true,
"started_at": "2026-11-20T08:20:40.887085261Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 500000000,
"ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "18093"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/usr/local/etc/haproxy/haproxy.cfg",
"name": null
}
]
},
{
"container": "712535508b37fe699452fb2a4ed62a005a8cfa7a14e2b9827a9c82cd4ec982c9",
"name": "/mtusi-lab6-loki-1",
"service": "loki",
"image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd",
"user": "10001",
"running": true,
"started_at": "2026-11-20T08:21:11.513864698Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 500000000,
"ports": {
"3100/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "13106"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/etc/loki/config.yaml",
"name": null
},
{
"type": "volume",
"destination": "/loki",
"name": "mtusi-lab6_logs_data"
}
]
},
{
"container": "6374cc6d3e2b163264c18e1181ea5be5b53f365a5324686a908ea8f94861a6c7",
"name": "/mtusi-lab6-migrate-1",
"service": "migrate",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": false,
"started_at": "2026-11-20T08:40:43.038416658Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 1000000000,
"ports": {},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/cache_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_api_secret",
"name": null
}
]
},
{
"container": "89012420d895c4c7c0d9fdaf7c1ff40c3a5f97dca279a0ef8cec0be3903dce42",
"name": "/mtusi-lab6-otel-collector-1",
"service": "otel-collector",
"image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:21:11.514654968Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 500000000,
"ports": {
"4317/tcp": null,
"4318/tcp": null,
"55679/tcp": null,
"8888/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "18896"
}
]
},
"mounts": [
{
"type": "bind",
"destination": "/etc/otel/config.yaml",
"name": null
}
]
},
{
"container": "2257079aebc7a27eda5607608d7c4e7df3d1a196eb2c65d7236a1b60f7466c91",
"name": "/mtusi-lab6-postgres-1",
"service": "postgres",
"image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
"user": "",
"running": true,
"started_at": "2026-11-20T08:20:26.86209481Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"5432/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/var/lib/postgresql/data",
"name": "mtusi-lab6_postgres_data"
},
{
"type": "bind",
"destination": "/run/secrets/postgres_password",
"name": null
}
]
},
{
"container": "937df26581a582233f77674d7c66a0b7fbba5fffd2377977ebd13857cd510a66",
"name": "/mtusi-lab6-prometheus-1",
"service": "prometheus",
"image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996",
"user": "nobody",
"running": true,
"started_at": "2026-11-20T08:21:11.515853797Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 268435456,
"nano_cpus": 500000000,
"ports": {
"9090/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "19096"
}
]
},
"mounts": [
{
"type": "volume",
"destination": "/prometheus",
"name": "mtusi-lab6_metrics_data"
},
{
"type": "bind",
"destination": "/etc/prometheus/prometheus.yaml",
"name": null
},
{
"type": "bind",
"destination": "/etc/prometheus/alerts.yaml",
"name": null
}
]
},
{
"container": "cc255a9bdb8967c853de2a4f167fe269a1a4f004e0b88ab174c79397212fd2a4",
"name": "/mtusi-lab6-redis-1",
"service": "redis",
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
"user": "",
"running": true,
"started_at": "2026-11-20T08:20:26.875269419Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 67108864,
"nano_cpus": 250000000,
"ports": {
"6379/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "db7892dbab7e8fc10b9d4431ee816406d604a540d7068b70aef208767168a178"
}
]
},
{
"container": "c4564618c7c266a67ed1f6a05989f8156ca42582a1860fe8d868fc758d18d4f3",
"name": "/mtusi-lab6-s3-1",
"service": "s3",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-11-20T08:20:31.57071865Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 500000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/s3_config",
"name": null
},
{
"type": "volume",
"destination": "/data",
"name": "3f039af87109799a85e80069eb418dd6c13b9b036ffc46fb0e6ed1bfc21407de"
}
]
},
{
"container": "d3abacb0bdaf348f062e5ec77599ea8a0450872a17b450b190ba7bce78166129",
"name": "/mtusi-lab6-s3-store-1",
"service": "s3-store",
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
"user": "",
"running": true,
"started_at": "2026-11-20T08:20:26.891493875Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 1000000000,
"ports": {
"18080/tcp": null,
"18888/tcp": null,
"19333/tcp": null,
"7333/tcp": null,
"8080/tcp": null,
"8333/tcp": null,
"8888/tcp": null,
"9333/tcp": null
},
"mounts": [
{
"type": "volume",
"destination": "/data",
"name": "mtusi-lab6_s3_data"
}
]
},
{
"container": "9cfc7a5053725737ecfc90211eed31a0c97a15aeb63ca270b9596c4af69de808",
"name": "/mtusi-lab6-tempo-1",
"service": "tempo",
"image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:28:48.715669396Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 536870912,
"nano_cpus": 500000000,
"ports": {
"3200/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "13206"
}
]
},
"mounts": [
{
"type": "volume",
"destination": "/var/tempo",
"name": "mtusi-lab6_traces_data"
},
{
"type": "bind",
"destination": "/etc/tempo/config.yaml",
"name": null
}
]
},
{
"container": "ef14a69e994f9f18fdb3438a03a28a974a3fcfc564f2d0eedeff3949a8f9bb29",
"name": "/mtusi-lab6-worker-1",
"service": "worker",
"image": "mtusi-messenger:lab6",
"user": "10001:10001",
"running": true,
"started_at": "2026-11-20T08:45:08.489572558Z",
"exit_code": 0,
"oom_killed": false,
"memory_limit": 805306368,
"nano_cpus": 1000000000,
"ports": {
"8080/tcp": null
},
"mounts": [
{
"type": "bind",
"destination": "/run/secrets/database_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/broker_url",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_key",
"name": null
},
{
"type": "bind",
"destination": "/run/secrets/s3_worker_secret",
"name": null
}
]
}
]
+989
View File
@@ -0,0 +1,989 @@
{
"trace_id": "dd1cb3e17dc547bc8bd3f94033e39f7a",
"retry_observed": true,
"trace": {
"batches": [
{
"resource": {
"attributes": [
{
"key": "deployment.environment.name",
"value": {
"stringValue": "lab"
}
},
{
"key": "service.instance.id",
"value": {
"stringValue": "067e0d5099614580ab61516259b76c14"
}
},
{
"key": "service.version",
"value": {
"stringValue": "lab6.1"
}
},
{
"key": "telemetry.sdk.language",
"value": {
"stringValue": "python"
}
},
{
"key": "telemetry.sdk.name",
"value": {
"stringValue": "opentelemetry"
}
},
{
"key": "telemetry.sdk.version",
"value": {
"stringValue": "1.36.0"
}
},
{
"key": "service.name",
"value": {
"stringValue": "messenger-api"
}
}
]
},
"scopeSpans": [
{
"scope": {
"name": "messenger"
},
"spans": [
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "k4UL1Qk8WSI=",
"parentSpanId": "ASNFZ4mrze8=",
"name": "POST /api/v1/chats/{chat_id}/attachments",
"kind": "SPAN_KIND_SERVER",
"startTimeUnixNano": "1795164308161361378",
"endTimeUnixNano": "1795164308169243229",
"attributes": [
{
"key": "http.request.method",
"value": {
"stringValue": "POST"
}
},
{
"key": "http.route",
"value": {
"stringValue": "/api/v1/chats/{chat_id}/attachments"
}
},
{
"key": "http.response.status_code",
"value": {
"intValue": "202"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "wgvjepi5Fj0=",
"parentSpanId": "k4UL1Qk8WSI=",
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164308161418698",
"endTimeUnixNano": "1795164308162079443",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "6f3FdmFjcfY=",
"parentSpanId": "k4UL1Qk8WSI=",
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164308162156009",
"endTimeUnixNano": "1795164308162662788",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "rvhnrnpClkQ=",
"parentSpanId": "k4UL1Qk8WSI=",
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164308163083649",
"endTimeUnixNano": "1795164308164393283",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "qBl9xvUTfq8=",
"parentSpanId": "k4UL1Qk8WSI=",
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164308164428226",
"endTimeUnixNano": "1795164308168921394",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "R2wwvDNpB68=",
"parentSpanId": "qBl9xvUTfq8=",
"name": "s3.put_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164308164676000",
"endTimeUnixNano": "1795164308168082844",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "put_object"
}
}
],
"status": {}
}
]
}
]
},
{
"resource": {
"attributes": [
{
"key": "deployment.environment.name",
"value": {
"stringValue": "lab"
}
},
{
"key": "service.instance.id",
"value": {
"stringValue": "6b019c7baab14c89976d2b4081f1c509"
}
},
{
"key": "service.version",
"value": {
"stringValue": "lab6.1"
}
},
{
"key": "telemetry.sdk.language",
"value": {
"stringValue": "python"
}
},
{
"key": "telemetry.sdk.name",
"value": {
"stringValue": "opentelemetry"
}
},
{
"key": "telemetry.sdk.version",
"value": {
"stringValue": "1.36.0"
}
},
{
"key": "service.name",
"value": {
"stringValue": "messenger-dispatcher"
}
}
]
},
"scopeSpans": [
{
"scope": {
"name": "messenger"
},
"spans": [
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "t4R0QxdjlSI=",
"parentSpanId": "qBl9xvUTfq8=",
"name": "attachment publish",
"kind": "SPAN_KIND_PRODUCER",
"startTimeUnixNano": "1795164308271913285",
"endTimeUnixNano": "1795164308274338175",
"attributes": [
{
"key": "messaging.system",
"value": {
"stringValue": "redis"
}
},
{
"key": "messaging.destination.name",
"value": {
"stringValue": "attachment-jobs"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "nBeLRAx4mXA=",
"parentSpanId": "qBl9xvUTfq8=",
"name": "attachment publish",
"kind": "SPAN_KIND_PRODUCER",
"startTimeUnixNano": "1795164312922355612",
"endTimeUnixNano": "1795164312924775467",
"attributes": [
{
"key": "messaging.system",
"value": {
"stringValue": "redis"
}
},
{
"key": "messaging.destination.name",
"value": {
"stringValue": "attachment-jobs"
}
}
],
"status": {}
}
]
}
]
},
{
"resource": {
"attributes": [
{
"key": "deployment.environment.name",
"value": {
"stringValue": "lab"
}
},
{
"key": "service.instance.id",
"value": {
"stringValue": "742be3a8d93342da9117f92711193f78"
}
},
{
"key": "service.version",
"value": {
"stringValue": "lab6.1"
}
},
{
"key": "telemetry.sdk.language",
"value": {
"stringValue": "python"
}
},
{
"key": "telemetry.sdk.name",
"value": {
"stringValue": "opentelemetry"
}
},
{
"key": "telemetry.sdk.version",
"value": {
"stringValue": "1.36.0"
}
},
{
"key": "service.name",
"value": {
"stringValue": "messenger-worker"
}
}
]
},
"scopeSpans": [
{
"scope": {
"name": "messenger"
},
"spans": [
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "AqkcRaZWl50=",
"parentSpanId": "t4R0QxdjlSI=",
"name": "attachment process",
"kind": "SPAN_KIND_CONSUMER",
"startTimeUnixNano": "1795164309031454501",
"endTimeUnixNano": "1795164312029923986",
"attributes": [
{
"key": "messaging.system",
"value": {
"stringValue": "redis"
}
},
{
"key": "messaging.destination.name",
"value": {
"stringValue": "attachment-jobs"
}
},
{
"key": "job.id",
"value": {
"stringValue": "9d711cdf-347d-4590-91da-4908beb37202"
}
},
{
"key": "job.attempt",
"value": {
"intValue": "1"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "cq34RtwooBc=",
"parentSpanId": "AqkcRaZWl50=",
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164309031484554",
"endTimeUnixNano": "1795164309038640365",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "PCx2LcCU6hc=",
"parentSpanId": "AqkcRaZWl50=",
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164309108621018",
"endTimeUnixNano": "1795164312013033788",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
},
{
"key": "error.type",
"value": {
"stringValue": "ReadTimeoutError"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "cX5pEW5zHb0=",
"parentSpanId": "AqkcRaZWl50=",
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164312013553635",
"endTimeUnixNano": "1795164312029457465",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "yG0WSHlT7kQ=",
"parentSpanId": "ccEcqG6ZxPo=",
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164312924989622",
"endTimeUnixNano": "1795164312936812921",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
}
]
}
]
},
{
"resource": {
"attributes": [
{
"key": "telemetry.sdk.language",
"value": {
"stringValue": "python"
}
},
{
"key": "telemetry.sdk.name",
"value": {
"stringValue": "opentelemetry"
}
},
{
"key": "telemetry.sdk.version",
"value": {
"stringValue": "1.36.0"
}
},
{
"key": "service.name",
"value": {
"stringValue": "messenger-worker"
}
},
{
"key": "service.version",
"value": {
"stringValue": "lab6.1"
}
},
{
"key": "service.instance.id",
"value": {
"stringValue": "742be3a8d93342da9117f92711193f78"
}
},
{
"key": "deployment.environment.name",
"value": {
"stringValue": "lab"
}
}
]
},
"scopeSpans": [
{
"scope": {
"name": "messenger"
},
"spans": [
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "ccEcqG6ZxPo=",
"parentSpanId": "nBeLRAx4mXA=",
"flags": 768,
"name": "attachment process",
"kind": "SPAN_KIND_CONSUMER",
"startTimeUnixNano": "1795164312924913496",
"endTimeUnixNano": "1795164314747321271",
"attributes": [
{
"key": "messaging.system",
"value": {
"stringValue": "redis"
}
},
{
"key": "messaging.destination.name",
"value": {
"stringValue": "attachment-jobs"
}
},
{
"key": "job.id",
"value": {
"stringValue": "9d711cdf-347d-4590-91da-4908beb37202"
}
},
{
"key": "job.attempt",
"value": {
"intValue": "2"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "Qf833YZm6d8=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164312943026506",
"endTimeUnixNano": "1795164314534106374",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "7QJGzOLYHEk=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.get_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314534253963",
"endTimeUnixNano": "1795164314538495180",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "get_object"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "uj8SpqBZjXM=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314714992685",
"endTimeUnixNano": "1795164314719977680",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
},
{
"key": "error.type",
"value": {
"stringValue": "ClientError"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "RPy3F6vP7SY=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.put_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314720094322",
"endTimeUnixNano": "1795164314725722348",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "put_object"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "HvB6dqnveCs=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314725791217",
"endTimeUnixNano": "1795164314727041344",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
},
{
"key": "error.type",
"value": {
"stringValue": "ClientError"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "DJqV5J+noKw=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.put_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314727130718",
"endTimeUnixNano": "1795164314729233570",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "put_object"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "YhhL1bvMeTM=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314729272549",
"endTimeUnixNano": "1795164314730313956",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
},
{
"key": "error.type",
"value": {
"stringValue": "ClientError"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "92rjoPWvm5g=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.put_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314730379104",
"endTimeUnixNano": "1795164314732329971",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "put_object"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "7VymNXkyejE=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314732386665",
"endTimeUnixNano": "1795164314733725616",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
},
{
"key": "error.type",
"value": {
"stringValue": "ClientError"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "5RGC0O8dQC8=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.put_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314733807464",
"endTimeUnixNano": "1795164314736051098",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "put_object"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "EGXBlCQ2GhE=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314736092741",
"endTimeUnixNano": "1795164314743975478",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
"spanId": "uADsu2p9U4s=",
"parentSpanId": "ccEcqG6ZxPo=",
"flags": 256,
"name": "s3.delete_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164314744065993",
"endTimeUnixNano": "1795164314746751040",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "delete_object"
}
}
],
"status": {}
}
]
}
]
}
]
},
"logs": [
{
"timestamp": "2026-11-20T08:45:12.027960+00:00",
"severity": "INFO",
"service.name": "messenger-worker",
"event": "job_retry",
"sampled": true,
"trace_id": "dd1cb3e17dc547bc8bd3f94033e39f7a",
"span_id": "717e69116e731dbd",
"job_id": "9d711cdf-347d-4590-91da-4908beb37202",
"attempt": 1,
"error_code": "storage_unavailable"
}
]
}
+348
View File
@@ -0,0 +1,348 @@
{
"method": "Three repeats per full/10%/25% setting; each holds sessions lock until pg_locks confirms it. 3x503 plus one slow200 per repeat. Sampling is ordinary SDK head sampling without forced traceparent.",
"runs": [
{
"mode": "full",
"repeat": 0,
"probes": [
{
"request_id": "95355e81cca54f83be9a19bd527e9825",
"status": 503,
"latency_seconds": 1.5030926670000007,
"trace_sampled": true,
"trace_id": "99c07cebfe6d37a278f4e0a9c603c1c2",
"tempo_trace_found": true
},
{
"request_id": "4e617be83d2247d0b849744fa0936deb",
"status": 503,
"latency_seconds": 1.6241221669999995,
"trace_sampled": true,
"trace_id": "806f738e3362110032113077494f8c8b",
"tempo_trace_found": true
},
{
"request_id": "b1c982a06765483eac67aa8cf72f2be1",
"status": 503,
"latency_seconds": 1.4874708749999996,
"trace_sampled": true,
"trace_id": "a4f07c609d396e2071bb8bdbebcbad4f",
"tempo_trace_found": true
},
{
"request_id": "822110e4f137423084355e5c8d260dcb",
"status": 200,
"latency_seconds": 1.1728969580000026,
"trace_sampled": true,
"trace_id": "8082c09a793e3535f33f203cb1dbe5da",
"tempo_trace_found": true
}
]
},
{
"mode": "full",
"repeat": 1,
"probes": [
{
"request_id": "da8b813d32d144cd844124a5fa86435e",
"status": 503,
"latency_seconds": 1.4984728329999975,
"trace_sampled": true,
"trace_id": "0fe73d0ff759f1e9f2f6d9e304a8fb96",
"tempo_trace_found": true
},
{
"request_id": "afb0eccacc314dcc8d15e6375ed1f674",
"status": 503,
"latency_seconds": 1.5027760410000006,
"trace_sampled": true,
"trace_id": "04c97d704d04c30ed712dc1a6c506639",
"tempo_trace_found": true
},
{
"request_id": "2633bfaaa83040dc9ec3f21c968b6774",
"status": 503,
"latency_seconds": 1.6313028749999994,
"trace_sampled": true,
"trace_id": "a849e96eb0258348dba259183e9455b5",
"tempo_trace_found": true
},
{
"request_id": "5f1a8b638d594e7e8e249d83e29a3c0a",
"status": 200,
"latency_seconds": 1.1764352089999974,
"trace_sampled": true,
"trace_id": "46f17923d4d83be8aec4863497913de9",
"tempo_trace_found": true
}
]
},
{
"mode": "full",
"repeat": 2,
"probes": [
{
"request_id": "f6cf9187274f4d4494f2546fdccf9cb5",
"status": 503,
"latency_seconds": 1.4932770829999953,
"trace_sampled": true,
"trace_id": "b57747b1e2c746aafa5fb7d7e9ec7e33",
"tempo_trace_found": true
},
{
"request_id": "5820d13764fe4ab5826a24e6be1dd12c",
"status": 503,
"latency_seconds": 1.5006451660000053,
"trace_sampled": true,
"trace_id": "f167d8d9df62e009ab0c2c3ac4ee3fc3",
"tempo_trace_found": true
},
{
"request_id": "18e12c0c4a424141b916fd4f7c847c8b",
"status": 503,
"latency_seconds": 1.5060892499999952,
"trace_sampled": true,
"trace_id": "453c386a3fcd16e940832672ef0b072a",
"tempo_trace_found": true
},
{
"request_id": "56df38255f974d75a68b6defb7ae7dbf",
"status": 200,
"latency_seconds": 1.181737209000005,
"trace_sampled": true,
"trace_id": "9380177eb739659ba6bfb122f4c5cb10",
"tempo_trace_found": true
}
]
},
{
"mode": "sample10",
"repeat": 0,
"probes": [
{
"request_id": "7c5131770a4e4ae78594065d30041f6b",
"status": 503,
"latency_seconds": 1.5020687499999994,
"trace_sampled": false,
"trace_id": "2f49be17fa517e268808bc7655dfec57",
"tempo_trace_found": false
},
{
"request_id": "083b4e2c54b34fb695e3e097774a02fb",
"status": 503,
"latency_seconds": 1.501452209000007,
"trace_sampled": false,
"trace_id": "6b7b3c7eb4f9d107cdef92d247489d39",
"tempo_trace_found": false
},
{
"request_id": "d63a94e59cc74918881a6c4821129fc0",
"status": 503,
"latency_seconds": 1.5036532090000065,
"trace_sampled": false,
"trace_id": "e6b3c5a1434815bc391a764eba18ad1f",
"tempo_trace_found": false
},
{
"request_id": "7aba69c8960a4e58ab8e1189f91e8df8",
"status": 200,
"latency_seconds": 1.1895337499999954,
"trace_sampled": true,
"trace_id": "a4b7c2e50c6db0db11b63d6988a5f581",
"tempo_trace_found": true
}
]
},
{
"mode": "sample10",
"repeat": 1,
"probes": [
{
"request_id": "db9a19fe6edf44b189e4c85527305b97",
"status": 503,
"latency_seconds": 1.508617583000003,
"trace_sampled": false,
"trace_id": "43478ce0a70fc97c1f551e0b770c8450",
"tempo_trace_found": false
},
{
"request_id": "259f18e925b54205a7f86ccf23588581",
"status": 503,
"latency_seconds": 1.6092364170000053,
"trace_sampled": false,
"trace_id": "de05c2c1da9da6e5aaad96f7562f8626",
"tempo_trace_found": false
},
{
"request_id": "7463cb267c1b4b7c836a79772d11d304",
"status": 503,
"latency_seconds": 1.4921612919999916,
"trace_sampled": false,
"trace_id": "63f2de04875064912b355e6f7ccf2bf8",
"tempo_trace_found": false
},
{
"request_id": "e6013482b7174a469182f9776832c5f8",
"status": 200,
"latency_seconds": 1.1786814999999962,
"trace_sampled": false,
"trace_id": "bc385f8114ac1d41fa55f762830d9a70",
"tempo_trace_found": false
}
]
},
{
"mode": "sample10",
"repeat": 2,
"probes": [
{
"request_id": "9a8de42292644772a11b19bb597b98e9",
"status": 503,
"latency_seconds": 1.4827315419999962,
"trace_sampled": false,
"trace_id": "62ef4444e74027412566b41ff3330a09",
"tempo_trace_found": false
},
{
"request_id": "444fd3c3d88c46cd89e1691852087fb3",
"status": 503,
"latency_seconds": 1.4932578330000013,
"trace_sampled": false,
"trace_id": "95d7610acd0f35487ec10e503fc12622",
"tempo_trace_found": false
},
{
"request_id": "6f616e3942134f7797c6c0b3767c0d0e",
"status": 503,
"latency_seconds": 1.497054833000007,
"trace_sampled": false,
"trace_id": "86793b238c9e2b7e586cb02e2f8bfc42",
"tempo_trace_found": false
},
{
"request_id": "718f724c1a8d41bfa3ec720daee432d3",
"status": 200,
"latency_seconds": 1.1874764580000061,
"trace_sampled": false,
"trace_id": "6b08c6339522e460bd77a9fd0f72438a",
"tempo_trace_found": false
}
]
},
{
"mode": "sample25",
"repeat": 0,
"probes": [
{
"request_id": "5c805a3468b64739bd1c839f6ddedd56",
"status": 503,
"latency_seconds": 1.4879085000000032,
"trace_sampled": false,
"trace_id": "5cf9d06acb989768f56a26c31e6d5255",
"tempo_trace_found": false
},
{
"request_id": "ff5596ad44c6461589e621a7b3f09676",
"status": 503,
"latency_seconds": 1.5042155419999972,
"trace_sampled": false,
"trace_id": "f8f790abc18c384d58225dceb4d1e8a9",
"tempo_trace_found": false
},
{
"request_id": "3ee8c8db9d4a4890b866d3cb40e18bb8",
"status": 503,
"latency_seconds": 1.4965452500000112,
"trace_sampled": false,
"trace_id": "b85f2a7d3cd2ff3dd4fa66fc8b37dea6",
"tempo_trace_found": false
},
{
"request_id": "f21c138335d64c688adb25c137c9a253",
"status": 200,
"latency_seconds": 1.1858503330000048,
"trace_sampled": true,
"trace_id": "c1df2d7ecfb8448a2f33be6756f23c38",
"tempo_trace_found": true
}
]
},
{
"mode": "sample25",
"repeat": 1,
"probes": [
{
"request_id": "053ece24cfa1465d9702c67781863703",
"status": 503,
"latency_seconds": 1.5010194580000018,
"trace_sampled": true,
"trace_id": "b7c73109f24a628720011f7401fc3d42",
"tempo_trace_found": true
},
{
"request_id": "7d30c3b85ebe480ea041d2261af90b3c",
"status": 503,
"latency_seconds": 1.4840848329999972,
"trace_sampled": false,
"trace_id": "438e9ec5155e621e5ef24df2b6e12568",
"tempo_trace_found": false
},
{
"request_id": "8c23bd6e2acc451d8abac7c787225b06",
"status": 503,
"latency_seconds": 1.4938515830000085,
"trace_sampled": false,
"trace_id": "92267b13ab39964ae58d8bf054c2f52a",
"tempo_trace_found": false
},
{
"request_id": "3ddb46344bae4d50989c8779264eeadf",
"status": 200,
"latency_seconds": 1.1738840829999901,
"trace_sampled": false,
"trace_id": "66d1a465ebede54e734068bc6e6092ca",
"tempo_trace_found": false
}
]
},
{
"mode": "sample25",
"repeat": 2,
"probes": [
{
"request_id": "dbcb7d27ff3a4067af49f2ad559beaf7",
"status": 503,
"latency_seconds": 1.4999495829999887,
"trace_sampled": false,
"trace_id": "88fa0b73db8d67b0be62f9ef653d5c1c",
"tempo_trace_found": false
},
{
"request_id": "1644972bdbbf4ad4ae0bb1be96d60853",
"status": 503,
"latency_seconds": 1.5033828749999998,
"trace_sampled": false,
"trace_id": "815ba97d46bc980d6df03e8542f1bd94",
"tempo_trace_found": false
},
{
"request_id": "34ecddeb28764f6da5aae311d8297f15",
"status": 503,
"latency_seconds": 1.486551832999993,
"trace_sampled": false,
"trace_id": "5b1d99439c3aadf44db722c9543be3f3",
"tempo_trace_found": false
},
{
"request_id": "558612f66b374e6d964b927a866bd9bf",
"status": 200,
"latency_seconds": 1.1901341250000144,
"trace_sampled": false,
"trace_id": "e6505908ffe05caf83a8a7d49f4476a0",
"tempo_trace_found": false
}
]
}
],
"presence_verification": "Actual Tempo trace API checked for every probe: sampled present; unsampled HTTP404."
}
+1
View File
@@ -0,0 +1 @@
{"instance": "208b43d0e3fe4d8daeccd4d252d691fe", "container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b", "running": false, "exit_code": 0, "seconds": 0.691}
+16
View File
@@ -0,0 +1,16 @@
{
"anonymous_object_and_list_403": true,
"job_fields": [
"attachment_id",
"attempt",
"created_at",
"job_id",
"pipeline_version",
"schema_version",
"traceparent",
"type"
],
"job_bytes": 317,
"no_source_bytes_or_credentials_in_envelope": true,
"strict_invalid_envelope_checks": true
}
+27
View File
@@ -0,0 +1,27 @@
{
"trace_id": "a630912396b64101b293cc306ee44ed8",
"historical_span_count": 21,
"historical_logs": 8,
"fixed_metric_query_time": 1795164311.5720682,
"metric_before": [
{
"metric": {},
"value": [
1795164311.572,
"485"
]
}
],
"metric_after": [
{
"metric": {},
"value": [
1795164311.572,
"485"
]
}
],
"business_data_and_existing_session_survived": true,
"all_telemetry_volumes_restarted": true,
"trace_log_metric_history_survived": true
}
+886
View File
@@ -0,0 +1,886 @@
{
"trace_id": "a630912396b64101b293cc306ee44ed8",
"request_id": "cb06618bb089444e8958f45beb2a8d1f",
"status": "ready",
"consumer_parent_matches_producer": true,
"producer_descends_from_upload": true,
"api_worker_logs_correlated": true,
"secret_canaries_absent": true,
"metric_labels_allowlisted": true,
"trace": {
"batches": [
{
"resource": {
"attributes": [
{
"key": "telemetry.sdk.language",
"value": {
"stringValue": "python"
}
},
{
"key": "telemetry.sdk.name",
"value": {
"stringValue": "opentelemetry"
}
},
{
"key": "telemetry.sdk.version",
"value": {
"stringValue": "1.36.0"
}
},
{
"key": "service.name",
"value": {
"stringValue": "messenger-dispatcher"
}
},
{
"key": "service.version",
"value": {
"stringValue": "lab6.1"
}
},
{
"key": "service.instance.id",
"value": {
"stringValue": "6b019c7baab14c89976d2b4081f1c509"
}
},
{
"key": "deployment.environment.name",
"value": {
"stringValue": "lab"
}
}
]
},
"scopeSpans": [
{
"scope": {
"name": "messenger"
},
"spans": [
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "1X47WVGmwVA=",
"parentSpanId": "XoCVQvOw3eM=",
"flags": 768,
"name": "attachment publish",
"kind": "SPAN_KIND_PRODUCER",
"startTimeUnixNano": "1795164306352857288",
"endTimeUnixNano": "1795164306353855758",
"attributes": [
{
"key": "messaging.system",
"value": {
"stringValue": "redis"
}
},
{
"key": "messaging.destination.name",
"value": {
"stringValue": "attachment-jobs"
}
}
],
"status": {}
}
]
}
]
},
{
"resource": {
"attributes": [
{
"key": "telemetry.sdk.language",
"value": {
"stringValue": "python"
}
},
{
"key": "telemetry.sdk.name",
"value": {
"stringValue": "opentelemetry"
}
},
{
"key": "telemetry.sdk.version",
"value": {
"stringValue": "1.36.0"
}
},
{
"key": "service.name",
"value": {
"stringValue": "messenger-worker"
}
},
{
"key": "service.version",
"value": {
"stringValue": "lab6.1"
}
},
{
"key": "service.instance.id",
"value": {
"stringValue": "c474b80aef4f42b1ba83aa52fc227420"
}
},
{
"key": "deployment.environment.name",
"value": {
"stringValue": "lab"
}
}
]
},
"scopeSpans": [
{
"scope": {
"name": "messenger"
},
"spans": [
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "jpUcOdrK9v8=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306353943301",
"endTimeUnixNano": "1795164306358874754",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "uqFoOGdo/j8=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306361469707",
"endTimeUnixNano": "1795164306363607190",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "Djfujh2+aLk=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.get_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306363643778",
"endTimeUnixNano": "1795164306365315189",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "get_object"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "UqSD5a1Hm0M=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306502165196",
"endTimeUnixNano": "1795164306504695870",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
},
{
"key": "error.type",
"value": {
"stringValue": "ClientError"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "FIWbTD1UmKU=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.put_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306504774140",
"endTimeUnixNano": "1795164306509272099",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "put_object"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "sGmfWvZhAhE=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306509328359",
"endTimeUnixNano": "1795164306510325785",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
},
{
"key": "error.type",
"value": {
"stringValue": "ClientError"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "cTvaoR1LX60=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.put_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306510386972",
"endTimeUnixNano": "1795164306512247503",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "put_object"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "n7+0IIgeeOo=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306512289102",
"endTimeUnixNano": "1795164306513273664",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
},
{
"key": "error.type",
"value": {
"stringValue": "ClientError"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "z5ewZlRmlSI=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.put_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306513330090",
"endTimeUnixNano": "1795164306515021299",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "put_object"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "ThQkryD54tU=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.head_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306515051329",
"endTimeUnixNano": "1795164306515871498",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "head_object"
}
},
{
"key": "error.type",
"value": {
"stringValue": "ClientError"
}
}
],
"status": {
"code": "STATUS_CODE_ERROR"
}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "3F5tg5VjfcM=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.put_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306515917149",
"endTimeUnixNano": "1795164306517590816",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "put_object"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "jRNKytkgcSc=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306517629032",
"endTimeUnixNano": "1795164306523308482",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "EPrL86X8HKM=",
"parentSpanId": "wUZSkS75RVs=",
"flags": 256,
"name": "s3.delete_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306523346489",
"endTimeUnixNano": "1795164306524869630",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "delete_object"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "wUZSkS75RVs=",
"parentSpanId": "1X47WVGmwVA=",
"flags": 768,
"name": "attachment process",
"kind": "SPAN_KIND_CONSUMER",
"startTimeUnixNano": "1795164306353917740",
"endTimeUnixNano": "1795164306525244943",
"attributes": [
{
"key": "messaging.system",
"value": {
"stringValue": "redis"
}
},
{
"key": "messaging.destination.name",
"value": {
"stringValue": "attachment-jobs"
}
},
{
"key": "job.id",
"value": {
"stringValue": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2"
}
},
{
"key": "job.attempt",
"value": {
"intValue": "1"
}
}
],
"status": {}
}
]
}
]
},
{
"resource": {
"attributes": [
{
"key": "telemetry.sdk.language",
"value": {
"stringValue": "python"
}
},
{
"key": "telemetry.sdk.name",
"value": {
"stringValue": "opentelemetry"
}
},
{
"key": "telemetry.sdk.version",
"value": {
"stringValue": "1.36.0"
}
},
{
"key": "service.name",
"value": {
"stringValue": "messenger-api"
}
},
{
"key": "service.version",
"value": {
"stringValue": "lab6.1"
}
},
{
"key": "service.instance.id",
"value": {
"stringValue": "067e0d5099614580ab61516259b76c14"
}
},
{
"key": "deployment.environment.name",
"value": {
"stringValue": "lab"
}
}
]
},
"scopeSpans": [
{
"scope": {
"name": "messenger"
},
"spans": [
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "s72qs5x19wo=",
"parentSpanId": "bWCElBq0RoM=",
"flags": 256,
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306279538296",
"endTimeUnixNano": "1795164306279912941",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "9hTiABG5UiA=",
"parentSpanId": "bWCElBq0RoM=",
"flags": 256,
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306279948610",
"endTimeUnixNano": "1795164306280391627",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "PPdidexZwfM=",
"parentSpanId": "bWCElBq0RoM=",
"flags": 256,
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306280710138",
"endTimeUnixNano": "1795164306281834242",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "PKeSMnEZomk=",
"parentSpanId": "XoCVQvOw3eM=",
"flags": 256,
"name": "s3.put_object",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306282049715",
"endTimeUnixNano": "1795164306285026998",
"attributes": [
{
"key": "rpc.system",
"value": {
"stringValue": "aws-api"
}
},
{
"key": "rpc.service",
"value": {
"stringValue": "S3"
}
},
{
"key": "rpc.method",
"value": {
"stringValue": "put_object"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "XoCVQvOw3eM=",
"parentSpanId": "bWCElBq0RoM=",
"flags": 256,
"name": "db.transaction",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1795164306281855668",
"endTimeUnixNano": "1795164306285749976",
"attributes": [
{
"key": "db.system",
"value": {
"stringValue": "postgresql"
}
}
],
"status": {}
},
{
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
"spanId": "bWCElBq0RoM=",
"parentSpanId": "ASNFZ4mrze8=",
"flags": 768,
"name": "POST /api/v1/chats/{chat_id}/attachments",
"kind": "SPAN_KIND_SERVER",
"startTimeUnixNano": "1795164306279505300",
"endTimeUnixNano": "1795164306285992974",
"attributes": [
{
"key": "http.request.method",
"value": {
"stringValue": "POST"
}
},
{
"key": "http.route",
"value": {
"stringValue": "/api/v1/chats/{chat_id}/attachments"
}
},
{
"key": "http.response.status_code",
"value": {
"intValue": "202"
}
}
],
"status": {}
}
]
}
]
}
]
},
"logs": [
{
"timestamp": "2026-11-20T08:45:06.285760+00:00",
"severity": "INFO",
"service.name": "messenger-api",
"event": "attachment_accepted",
"sampled": true,
"trace_id": "a630912396b64101b293cc306ee44ed8",
"span_id": "6d6084941ab44683",
"attachment_id": "a6e6251c-5774-4a3b-8424-c72d9568e4a6",
"job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2"
},
{
"timestamp": "2026-11-20T08:45:06.525174+00:00",
"severity": "INFO",
"service.name": "messenger-worker",
"event": "job_processed",
"sampled": true,
"trace_id": "a630912396b64101b293cc306ee44ed8",
"span_id": "c14652912ef9455b",
"job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2",
"attempt": 1,
"outcome": "ready"
},
{
"timestamp": "2026-11-20T08:45:06.353780+00:00",
"severity": "INFO",
"service.name": "messenger-dispatcher",
"event": "job_published",
"sampled": true,
"trace_id": "a630912396b64101b293cc306ee44ed8",
"span_id": "d57e3b5951a6c150",
"job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2",
"attempt": 1
},
{
"timestamp": "2026-11-20T08:45:06.285952+00:00",
"severity": "INFO",
"service.name": "messenger-api",
"event": "http_request",
"sampled": true,
"trace_id": "a630912396b64101b293cc306ee44ed8",
"span_id": "6d6084941ab44683",
"request_id": "cb06618bb089444e8958f45beb2a8d1f",
"method": "POST",
"route": "/api/v1/chats/{chat_id}/attachments",
"status": 202,
"duration_seconds": 0.006511
},
{
"timestamp": "2026-11-20T08:45:06.504597+00:00",
"severity": "INFO",
"service.name": "messenger-worker",
"event": "operation_failed",
"sampled": true,
"trace_id": "a630912396b64101b293cc306ee44ed8",
"span_id": "52a483e5ad479b43",
"error_type": "ClientError"
},
{
"timestamp": "2026-11-20T08:45:06.510258+00:00",
"severity": "INFO",
"service.name": "messenger-worker",
"event": "operation_failed",
"sampled": true,
"trace_id": "a630912396b64101b293cc306ee44ed8",
"span_id": "b0699f5af6610211",
"error_type": "ClientError"
},
{
"timestamp": "2026-11-20T08:45:06.513218+00:00",
"severity": "INFO",
"service.name": "messenger-worker",
"event": "operation_failed",
"sampled": true,
"trace_id": "a630912396b64101b293cc306ee44ed8",
"span_id": "9fbfb420881e78ea",
"error_type": "ClientError"
},
{
"timestamp": "2026-11-20T08:45:06.515824+00:00",
"severity": "INFO",
"service.name": "messenger-worker",
"event": "operation_failed",
"sampled": true,
"trace_id": "a630912396b64101b293cc306ee44ed8",
"span_id": "4e1424af20f9e2d5",
"error_type": "ClientError"
}
]
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 84 KiB

+67
View File
@@ -0,0 +1,67 @@
# Локальный runbook — lab6
Grafana: http://127.0.0.1:13006/d/messenger-lab6.
Prometheus: http://127.0.0.1:19096.
Loki и Tempo доступны через Explore и provisioned datasources.
Не запрашивать credentials, HTTP bodies, raw URL или SQL parameters для диагностики.
## api-errors
1. Открыть API throughput, 5xx ratio и p95/p99. Health исключён.
2. Проверить APIHighErrorRate в Prometheus /alerts: 5xx / все обработанные business requests за 30s > 5%, for 6s.
3. В Loki: {service_name="messenger-api"} | json | status >= 500.
4. Взять trace_id из лога и перейти в Tempo. X-Request-Id связывает ответ с тем же логом.
5. Если основное время/ошибка в db.transaction, проверить db_pool_used/waiting и QueueObservationUnavailable:
проблема на границе PostgreSQL/пула, а не декодирования изображения.
6. Контролируемый инцидент теста удерживает ACCESS EXCLUSIVE lock таблицы sessions: lock_timeout=1.5s даёт 503.
SELECT 1 readiness остаётся доступным; долгий span db.transaction показывает причину, даже если HTTP health зелёный.
7. После снятия блокировки тот же Bearer снова работает. Подождать очистки 30s окна и for; проверить resolved.
## queue-lag
1. Проверить queue_depth и queue_oldest_age_seconds: это принятые незавершённые jobs в PostgreSQL,
включая ещё не опубликованный outbox; это не длина Redis stream и не длительность resize.
2. Если upload остаётся 202, а возраст/глубина растут, проверить worker busy/capacity и отсутствие новых job_processed.
Удалённый процесс может оставаться в scrape cache до 30s; учитывать эту задержку.
3. В Loki: {service_name=~"messenger-.*"} | json | event=~"attachment_accepted|job_published|job_processed|job_retry".
4. Через trace_id пройти API → producer → consumer → S3/DB.
Published без consumer указывает на отсутствие/недоступность worker или его чтения broker.
Consumer с retry и ошибочным S3 span указывает на storage, а не потерю задания.
5. Восстановить worker. Проверить ready, уменьшение backlog до нуля, исчезновение QueueLag.
6. Не удалять volume, stream, job registry или staging ради зелёного dashboard.
## telemetry-backend
1. Если business API продолжает отвечать, но trace не находится, проверить Collector:
otelcol_exporter_send_failed_spans_total, otelcol_exporter_queue_size/capacity.
2. Уточнить, принимает ли Collector OTLP (otelcol_receiver_accepted_spans_total).
Ошибки exporter при растущем accepted означают проблему downstream, а не отсутствие инструментации.
3. Восстановить Tempo/Loki; проверить новый trace/log. Старые telemetry, отброшенные после 10s retry budget,
не обязаны появиться. Бизнес-данные и telemetry имеют разные гарантии сохранности.
4. Collector: memory limiter 192MiB, batch <=256, две exporter-очереди по 128 batches;
SDK: span queue512/batch128, logs256/batch64, OTLP timeout1s.
Нет бесконечной очереди и обязательной блокирующей синхронной отправки на HTTP-path.
5. У каждого backend именованный volume. Отказ процесса и потеря диска — разные инциденты.
## SLI / SLO
Учебное окно 5 минут, при наличии запросов:
- API availability SLI = 1 - sum(increase(http_server_requests_total{status_class="5xx"}[5m]))
/ sum(increase(http_server_requests_total[5m])). Цель >=99%.
4xx считаются корректно обслуженными запросами, health исключён.
Это SLI обработанных API запросов: отказ LB/сети до API в знаменатель не попадает.
- Time-to-ready latency SLI среди завершённых ready jobs:
sum(increase(job_ready_duration_seconds_bucket{le="5"}[5m]))
/ sum(increase(job_ready_duration_seconds_count[5m])). Цель >=95%.
Не завершённые/failed jobs не скрываются: отдельно jobs_processed outcomes, job_retries и QueueLag.
Этот latency SLO не является SLO доступности всего pipeline.
Короткие алерты 30s и 10s queue age предназначены для локальной демонстрации.
Они не заявлены промышленными burn-rate alerts; выбран трек 1.
Balanced default: TRACE_SAMPLE_RATIO=0.25; metrics/logs всегда включены.
У несэмплированного запроса trace_id присутствует в log, но полного trace в Tempo не будет.
Для контролируемого proof используется W3C sampled=1; для полной локальной диагностики:
TRACE_SAMPLE_RATIO=1 docker compose up -d --no-deps api worker dispatcher.
Это не tail sampling: ошибки и медленные spans не получают автоматического приоритета.
+27
View File
@@ -0,0 +1,27 @@
groups:
- name: messenger
rules:
- alert: APIHighErrorRate
expr: sum(rate(http_server_requests_total{service_name="messenger-api",status_class="5xx"}[30s])) / clamp_min(sum(rate(http_server_requests_total{service_name="messenger-api"}[30s])), 0.001) > 0.05
for: 6s
labels:
severity: warning
annotations:
summary: API 5xx share exceeds 5 percent in the last 30 seconds
runbook: infra/RUNBOOK.md#api-errors
- alert: QueueLag
expr: max(queue_oldest_age_seconds{service_name="messenger-dispatcher"}) > 10 and max(queue_depth{service_name="messenger-dispatcher"}) > 0
for: 6s
labels:
severity: warning
annotations:
summary: Accepted unfinished image work older than 10 seconds
runbook: infra/RUNBOOK.md#queue-lag
- alert: QueueObservationUnavailable
expr: min(queue_observation_ok{service_name="messenger-dispatcher"}) == 0
for: 6s
labels:
severity: warning
annotations:
summary: Dispatcher cannot observe PostgreSQL backlog
runbook: infra/RUNBOOK.md#api-errors
+713
View File
@@ -0,0 +1,713 @@
{
"uid": "messenger-lab6",
"title": "MTUSI lab6 — Messenger observability",
"schemaVersion": 41,
"version": 1,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"tags": [
"mtusi",
"lab6"
],
"annotations": {
"list": []
},
"panels": [
{
"id": 1,
"title": "API throughput (health excluded)",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(http_server_requests_total{service_name=\"messenger-api\"}[1m]))",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 2,
"title": "API 5xx ratio",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(http_server_requests_total{service_name=\"messenger-api\",status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{service_name=\"messenger-api\"}[1m])),0.001)",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 3,
"title": "API p50 from histogram",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 4,
"title": "API p95 from histogram",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 5,
"title": "API p99 from histogram",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 6,
"title": "Worker jobs by outcome",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (outcome) (rate(jobs_processed_total{service_name=\"messenger-worker\"}[1m]))",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 7,
"title": "Worker processing p95",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 8,
"title": "Accepted to ready p95",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95,sum by(le)(rate(job_ready_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 9,
"title": "Accepted unfinished jobs",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max(queue_depth{service_name=\"messenger-dispatcher\"})",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 10,
"title": "Oldest unfinished job age",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max(queue_oldest_age_seconds{service_name=\"messenger-dispatcher\"})",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 11,
"title": "DB pool used",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by(service_name) (db_pool_used{service_name=\"messenger-api\"})",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 12,
"title": "DB pool waiting",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by(service_name) (db_pool_waiting{service_name=\"messenger-api\"})",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 13,
"title": "Worker busy / capacity",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 14,
"title": "Process resident memory",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "process_resident_memory_bytes{service_name=~\"messenger-.*\"}",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 15,
"title": "Process CPU seconds per second",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by(service_name)(rate(process_cpu_seconds_total{service_name=~\"messenger-.*\"}[1m]))",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 16,
"title": "Container memory incl renderer",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "container_memory_bytes{service_name=~\"messenger-.*\"}",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 17,
"title": "Local alerts",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 64,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "ALERTS{alertstate=\"firing\"}",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 18,
"title": "Collector failed span exports",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 64,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))",
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
}
],
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 100,
"title": "Correlated structured logs: open TraceID for API / publish / worker chain",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"x": 0,
"y": 72,
"w": 24,
"h": 10
},
"targets": [
{
"refId": "A",
"expr": "{service_name=~\"messenger-.*\"} | json",
"queryType": "range"
}
],
"options": {
"showTime": true,
"showLabels": false,
"sortOrder": "Descending"
}
},
{
"id": 101,
"title": "Terminal failed jobs",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 82,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(job_retries_total{service_name=\"messenger-worker\",outcome=\"failed\"}[1m]))"
}
]
},
{
"id": 102,
"title": "Retry attempts",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 82,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(job_retries_total{service_name=\"messenger-worker\",outcome=\"retry\"}[1m]))"
}
]
}
]
}
@@ -0,0 +1,8 @@
apiVersion: 1
providers:
- name: Messenger
folder: MTUSI
type: file
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,32 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: TraceID
datasourceUid: tempo
matcherRegex: '"trace_id":"([a-f0-9]{32})"'
url: '$${__value.raw}'
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: '-1m'
spanEndTimeShift: '1m'
filterByTraceID: false
customQuery: true
query: '{service_name=~"messenger-.*"} | json | trace_id="$${__trace.traceId}"'
+46
View File
@@ -0,0 +1,46 @@
global
maxconn 1024
nbthread 1
hard-stop-after 10s
defaults
mode http
timeout connect 500ms
timeout check 500ms
timeout queue 1s
timeout client 30s
timeout server 30s
timeout http-request 3s
retries 2
retry-on conn-failure
option http-server-close
resolvers docker
nameserver docker_dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 2
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold nx 1s
hold refused 1s
hold timeout 1s
hold other 1s
hold obsolete 1s
frontend public
bind :8080
default_backend replicas
backend replicas
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fall 2 rise 2
server-template api 1-4 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
listen metrics
bind :8404
stats enable
stats uri /stats
+33
View File
@@ -0,0 +1,33 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 24h
ingestion_rate_mb: 4
ingestion_burst_size_mb: 8
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+66
View File
@@ -0,0 +1,66 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 0.0.0.0:8889
metric_expiration: 30s
resource_to_telemetry_conversion:
enabled: true
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 2s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 128
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 3s
max_elapsed_time: 10s
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 128
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 3s
max_elapsed_time: 10s
service:
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
+11
View File
@@ -0,0 +1,11 @@
global:
scrape_interval: 2s
evaluation_interval: 2s
rule_files: [/etc/prometheus/alerts.yaml]
scrape_configs:
- job_name: messenger
static_configs:
- targets: [otel-collector:8889]
- job_name: collector
static_configs:
- targets: [otel-collector:8888]
+24
View File
@@ -0,0 +1,24 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
ingester:
trace_idle_period: 5s
max_block_duration: 1m
max_block_bytes: 1048576
compactor:
compaction:
block_retention: 24h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+42
View File
@@ -0,0 +1,42 @@
CREATE TABLE users (
id UUID PRIMARY KEY
, username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$')
, display_name TEXT NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100)
, password_hash TEXT NOT NULL
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
)
;
CREATE TABLE chats (
id UUID PRIMARY KEY
, title TEXT NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100)
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, last_message_at TIMESTAMPTZ
)
;
CREATE TABLE chat_members (
chat_id UUID NOT NULL REFERENCES chats(id)
, user_id UUID NOT NULL REFERENCES users(id)
, PRIMARY KEY (chat_id, user_id)
)
;
CREATE INDEX chat_members_user ON chat_members (user_id, chat_id)
;
CREATE TABLE messages (
id UUID PRIMARY KEY
, chat_id UUID NOT NULL REFERENCES chats(id)
, sender_id UUID NOT NULL REFERENCES users(id)
, text TEXT NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000)
, created_at TIMESTAMPTZ NOT NULL
, FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
)
;
CREATE INDEX messages_history ON messages (chat_id, created_at, id)
;
CREATE TABLE sessions (
token_hash TEXT PRIMARY KEY
, user_id UUID NOT NULL REFERENCES users(id)
, expires_at TIMESTAMPTZ NOT NULL
)
;
CREATE INDEX sessions_expiry ON sessions (expires_at)
;
+17
View File
@@ -0,0 +1,17 @@
-- PostgreSQL TEXT cannot contain U+0000. Store validated UTF-8 losslessly in BYTEA.
-- API validation keeps the contract's code-point limits; these bounds cap bytes.
ALTER TABLE users
DROP CONSTRAINT users_display_name_check
, ALTER COLUMN display_name TYPE BYTEA USING convert_to(display_name, 'UTF8')
, ADD CONSTRAINT users_display_name_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400)
;
ALTER TABLE chats
DROP CONSTRAINT chats_title_check
, ALTER COLUMN title TYPE BYTEA USING convert_to(title, 'UTF8')
, ADD CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400)
;
ALTER TABLE messages
DROP CONSTRAINT messages_text_check
, ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8')
, ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000)
;
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE idempotency_keys (
user_id UUID NOT NULL REFERENCES users(id)
, chat_id UUID NOT NULL REFERENCES chats(id)
, key VARCHAR(128) NOT NULL
, request_hash CHAR(64) NOT NULL
, message_id UUID REFERENCES messages(id)
, expires_at TIMESTAMPTZ NOT NULL
, PRIMARY KEY (user_id, chat_id, key)
)
;
CREATE INDEX idempotency_expiry ON idempotency_keys (expires_at)
;
+30
View File
@@ -0,0 +1,30 @@
ALTER TABLE messages
DROP CONSTRAINT messages_text_bytes
, ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000)
, ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}'
;
CREATE TABLE attachments (
id UUID PRIMARY KEY
, chat_id UUID NOT NULL REFERENCES chats(id)
, owner_id UUID NOT NULL REFERENCES users(id)
, filename TEXT NOT NULL
, object_key TEXT NOT NULL UNIQUE
, content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg'))
, size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760)
, sha256 CHAR(64) NOT NULL
, state TEXT NOT NULL CHECK (state IN ('uploading', 'ready', 'storage_missing'))
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id)
)
;
CREATE INDEX attachments_chat ON attachments (chat_id, id)
;
CREATE TABLE message_attachments (
message_id UUID NOT NULL REFERENCES messages(id)
, attachment_id UUID NOT NULL REFERENCES attachments(id)
, PRIMARY KEY (message_id, attachment_id)
)
;
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id)
;
+40
View File
@@ -0,0 +1,40 @@
ALTER TABLE attachments
DROP CONSTRAINT attachments_state_check
, ADD CONSTRAINT attachments_state_check CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed', 'storage_missing'))
, ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1
, ADD COLUMN staging_key TEXT
, ADD COLUMN variants JSONB NOT NULL DEFAULT '[]'
, ADD COLUMN error_code TEXT
, ADD COLUMN current_job_id UUID
;
CREATE TABLE attachment_jobs (
id UUID PRIMARY KEY
, attachment_id UUID NOT NULL REFERENCES attachments(id)
, pipeline_version INTEGER NOT NULL
, source_key TEXT NOT NULL
, state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded'))
, attempt INTEGER NOT NULL DEFAULT 0
, max_attempts INTEGER NOT NULL DEFAULT 5
, lease_token UUID
, lease_until TIMESTAMPTZ
, next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, published_at TIMESTAMPTZ
, error_code TEXT
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
, UNIQUE (attachment_id, pipeline_version)
)
;
ALTER TABLE attachments
ADD CONSTRAINT attachments_current_job FOREIGN KEY (current_job_id) REFERENCES attachment_jobs(id)
;
CREATE INDEX jobs_due ON attachment_jobs (state, next_attempt_at)
;
CREATE TABLE job_events (
id BIGSERIAL PRIMARY KEY
, job_id UUID NOT NULL REFERENCES attachment_jobs(id)
, event TEXT NOT NULL
, attempt INTEGER NOT NULL
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
)
;
+3
View File
@@ -0,0 +1,3 @@
ALTER TABLE attachment_jobs
ADD COLUMN trace_context JSONB NOT NULL DEFAULT '{}'
;
+27
View File
@@ -0,0 +1,27 @@
boto3==1.40.61
botocore==1.40.76
certifi==2026.10.23
charset-normalizer==3.5.2
googleapis-common-protos==1.76.0
idna==3.20
importlib_metadata==8.7.1
jmespath==1.1.0
opentelemetry-api==1.36.0
opentelemetry-exporter-otlp-proto-common==1.36.0
opentelemetry-exporter-otlp-proto-http==1.36.0
opentelemetry-proto==1.36.0
opentelemetry-sdk==1.36.0
opentelemetry-semantic-conventions==0.57b0
pillow==13.0.0
protobuf==6.33.6
psycopg==3.2.3
psycopg-binary==3.2.3
psycopg-pool==3.2.4
python-dateutil==2.9.0.post0
redis==5.2.1
requests==2.34.2
s3transfer==0.14.0
six==1.17.0
typing_extensions==4.16.0
urllib3==2.8.0
zipp==4.1.2
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Create local credentials without printing or overwriting existing values."""
import json
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1]
directory = root / 'secrets'
directory.mkdir(mode=0o700, exist_ok=True)
def save(name, value):
path = directory / name
if not path.exists():
path.write_text(value)
path.chmod(0o644) # UID 10001 in Compose; parent is owner-only 0700.
return path.read_text().strip()
password = save('postgres_password', secrets.token_hex(24))
save('database_url', 'postgresql://messenger:' + password + '@postgres:5432/messenger')
save('cache_url', 'redis://redis:6379/0')
save('broker_url', 'redis://broker:6379/0')
identities = []
for role, actions in {
'admin': ['Admin', 'Read', 'Write', 'List'],
'api': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'],
'worker': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'],
}.items():
key = save('s3_' + role + '_key', secrets.token_hex(12))
secret = save('s3_' + role + '_secret', secrets.token_hex(24))
identities.append({'name': role, 'credentials': [{'accessKey': key, 'secretKey': secret}], 'actions': actions})
configuration = directory / 's3_config'
configuration.write_text(json.dumps({'identities': identities}))
configuration.chmod(0o600) # Only the root S3 gateway mounts the full identity configuration.
env = root / '.env'
if not env.exists():
env.write_text((root / '.env.example').read_text())
env.chmod(0o600)
print('Local configuration ready; no secrets printed.')
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
python3 scripts/snapshot.py > evidence/restart-before.json
docker compose stop api lb worker dispatcher
docker compose restart postgres redis s3-store s3 broker otel-collector prometheus loki tempo grafana
docker compose up -d --wait postgres redis s3-store s3 broker otel-collector prometheus loki tempo grafana
docker compose up -d --wait api lb worker dispatcher
python3 scripts/snapshot.py > evidence/restart-after.json
+11
View File
@@ -0,0 +1,11 @@
"""Sanitized process and resource evidence; never dump container environments."""
import json
import subprocess
ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split()
records = []
for container in ids:
raw = subprocess.check_output(['docker', 'inspect', container], text=True)
data = json.loads(raw)[0]
records.append({'container': data['Id'], 'name': data['Name'], 'service': data['Config']['Labels']['com.docker.compose.service'], 'image': data['Config']['Image'], 'user': data['Config']['User'], 'running': data['State']['Running'], 'started_at': data['State']['StartedAt'], 'exit_code': data['State']['ExitCode'], 'oom_killed': data['State']['OOMKilled'], 'memory_limit': data['HostConfig']['Memory'], 'nano_cpus': data['HostConfig']['NanoCpus'], 'ports': data['NetworkSettings']['Ports'], 'mounts': [{'type': m['Type'], 'destination': m['Destination'], 'name': m.get('Name')} for m in data['Mounts']]})
print(json.dumps(records, indent=2))
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop_one.py
+32
View File
@@ -0,0 +1,32 @@
"""Resolve the requested process alias using each actual container, never a fake header."""
import json
import os
from pathlib import Path
import subprocess
import time
def docker(*args):
return subprocess.check_output(['docker', *args], text=True).strip()
def instance(container):
code = "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=1).headers['X-Instance-Id'])"
return docker('exec', container, 'python', '-c', code)
if __name__ == '__main__':
target = os.environ['TARGET_INSTANCE_ID']
started = time.monotonic()
for container in docker('compose', 'ps', '-q', 'api').splitlines():
if instance(container) == target:
docker('stop', '--time', '8', container)
state = json.loads(docker('inspect', '--format', '{{json .State}}', container))
assert not state['Running'], 'target must remain stopped'
record = {'instance': target, 'container': container, 'running': state['Running'], 'exit_code': state['ExitCode'], 'seconds': round(time.monotonic() - started, 3)}
with Path('evidence/stopped-instances.jsonl').open('a') as out:
out.write(json.dumps(record) + '\n')
print(json.dumps(record))
break
else:
raise SystemExit('Requested live API instance not found')
+16
View File
@@ -0,0 +1,16 @@
import os
import time
import urllib.request
base = os.environ.get('BASE_URL', 'http://127.0.0.1:18087')
for attempt in range(120):
try:
with urllib.request.urlopen(base + '/health/ready', timeout=2) as response:
if response.status == 200:
print('Ready through load balancer.')
break
except OSError:
pass
time.sleep(0.5)
else:
raise SystemExit('Readiness timeout')
+127
View File
@@ -0,0 +1,127 @@
"""Explicit local operator tools; no administrative HTTP endpoint."""
import argparse
import json
import os
import uuid
from pipeline import connect, audit
from objects import Objects
from postgres import setting
def replay(job_id):
with connect() as conn, conn.transaction():
job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job_id,)).fetchone()
if job is None or job['state'] != 'failed':
raise ValueError('Replay requires a failed job')
attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND current_job_id = %s FOR UPDATE', (job_id,)).fetchone()
if attachment is None or attachment['state'] != 'failed' or Objects(setting).head(job['source_key']) is None:
raise ValueError('Current failed source is unavailable')
conn.execute('''
UPDATE attachment_jobs
SET state = 'queued'
, max_attempts = attempt + %s
, published_at = NULL
, next_attempt_at = clock_timestamp()
, error_code = NULL
, updated_at = clock_timestamp()
WHERE 1=1
AND id = %s
''', (int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), job_id))
conn.execute("UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job_id,))
audit(conn, job, 'operator_replay')
return {'job_id': str(job_id), 'replayed': True}
def version(attachment_id, number):
if number not in (1, 2):
raise ValueError('Only implemented pipeline versions 1 and 2 are supported')
with connect() as conn, conn.transaction():
row = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone()
if row is None or number != row['pipeline_version'] + 1 or row['state'] in ('uploading', 'failed', 'storage_missing'):
raise ValueError('Version must advance the current process exactly once')
old = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (row['current_job_id'],)).fetchone()
source = row['object_key'] if row['state'] == 'ready' else old['source_key']
if Objects(setting).head(source) is None:
raise ValueError('Source unavailable')
conn.execute("UPDATE attachment_jobs SET state = 'superseded', lease_token = NULL, lease_until = NULL WHERE 1=1 AND id = %s", (old['id'],))
audit(conn, old, 'operator_supersede')
job_id = uuid.uuid4()
conn.execute('''
INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts)
VALUES (%s, %s, %s, %s, %s)
''', (job_id, attachment_id, number, source, int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))))
conn.execute('''
UPDATE attachments
SET pipeline_version = %s
, current_job_id = %s
, state = 'queued'
, variants = '[]'
, error_code = NULL
, updated_at = clock_timestamp()
WHERE 1=1
AND id = %s
''', (number, job_id, attachment_id))
return {'job_id': str(job_id), 'version': number}
def cleanup(apply=False, retention=86400):
if retention < 86400:
raise ValueError('Retention must be >=24 hours')
objects = Objects(setting)
counts = {'ready_staging': 0, 'expired_staging': 0}
with connect() as conn:
# Intent rows cover every valid staging PUT. Never collect active jobs.
last = '00000000-0000-0000-0000-000000000000'
while True:
rows = conn.execute('SELECT id FROM attachments WHERE 1=1 AND id > %s ORDER BY id LIMIT 100', (last,)).fetchall()
if not rows:
break
for candidate in rows:
last = candidate['id']
with conn.transaction():
row = conn.execute('''
SELECT *
FROM attachments
WHERE 1=1
AND id = %s
AND (state = 'ready' OR (state IN ('uploading', 'failed') AND updated_at < clock_timestamp() - %s * INTERVAL '1 second'))
FOR UPDATE SKIP LOCKED
''', (last, retention)).fetchone()
if row is None or not row['staging_key'] or not row['staging_key'].startswith('staging/'):
continue
counts['ready_staging' if row['state'] == 'ready' else 'expired_staging'] += 1
if apply:
objects.delete(row['staging_key'])
if row['state'] == 'uploading' and row['current_job_id'] is None:
conn.execute('DELETE FROM attachments WHERE 1=1 AND id = %s', (last,))
else:
conn.execute('UPDATE attachments SET staging_key = NULL WHERE 1=1 AND id = %s', (last,))
return dict(apply=apply, retention_seconds=retention, counts=counts)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('action', choices=['failed', 'replay', 'version', 'cleanup'])
p.add_argument('identifier', nargs='?')
p.add_argument('--version', type=int)
p.add_argument('--apply', action='store_true')
args = p.parse_args()
try:
if args.action == 'failed':
with connect() as conn:
result = conn.execute("SELECT id, attachment_id, pipeline_version, attempt, max_attempts, error_code FROM attachment_jobs WHERE 1=1 AND state = 'failed' ORDER BY created_at LIMIT 100").fetchall()
elif args.action == 'replay':
result = replay(uuid.UUID(args.identifier))
elif args.action == 'version':
result = version(uuid.UUID(args.identifier), args.version)
else:
result = cleanup(args.apply)
print(json.dumps(result, default=str, indent=2))
except Exception:
print(json.dumps({'error': 'Operator action refused or dependency unavailable; no secrets logged.'}))
raise SystemExit(1)
if __name__ == '__main__':
main()
+13
View File
@@ -0,0 +1,13 @@
"""Idempotent private bucket bootstrap, executed only with administrative S3 keys."""
from botocore.exceptions import ClientError
from objects import Objects
from postgres import setting
objects = Objects(setting)
try:
objects.client.create_bucket(Bucket=objects.bucket)
except ClientError as exc:
if exc.response['Error']['Code'] not in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
raise RuntimeError('Bucket bootstrap failed') from None
objects.ready()
print('Private bucket ready; static authenticated identities loaded from local secret config.')
+98
View File
@@ -0,0 +1,98 @@
"""Shared HTTP contract; storage is supplied by each laboratory."""
import base64
import datetime as dt
import json
import re
import uuid
class APIError(Exception):
def __init__(self, status, code, message):
self.status, self.code, self.message = status, code, message
def invalid():
raise APIError(400, 'invalid_request', 'Неверные параметры запроса')
def unauthorized():
raise APIError(401, 'unauthorized', 'Требуется действительная идентификация')
def missing():
raise APIError(404, 'not_found', 'Объект не найден')
def identifier(value):
try:
if not isinstance(value, str):
invalid()
return str(uuid.UUID(value))
except ValueError:
invalid()
def text(value, maximum, minimum=1):
if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
invalid()
try:
value.encode('utf-8')
except UnicodeError:
invalid()
return value
def fields(value, required):
if not isinstance(value, dict) or set(value) != set(required):
invalid()
def username(value):
if not isinstance(value, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', value):
invalid()
return value
def now():
return dt.datetime.now(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def page_args(query, scope):
if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()):
invalid()
limit = query.get('limit', ['50'])[0]
if not re.fullmatch(r'[0-9]+', limit):
invalid()
try:
limit = int(limit)
except ValueError:
invalid()
if not 1 <= limit <= 100:
invalid()
cursor = None
if 'cursor' in query:
raw = query['cursor'][0]
try:
if not raw or len(raw) > 1024:
invalid()
decoded = json.loads(base64.b64decode(raw + '=' * (-len(raw) % 4), altchars=b'-_', validate=True))
if not isinstance(decoded, list) or len(decoded) != 3 or decoded[0] != scope:
invalid()
stamp = decoded[1]
if not isinstance(stamp, str) or not stamp.endswith('Z'):
invalid()
stamp = dt.datetime.fromisoformat(stamp[:-1] + '+00:00').isoformat(timespec='microseconds').replace('+00:00', 'Z')
cursor = (stamp, identifier(decoded[2]))
except (ValueError, TypeError, UnicodeError):
invalid()
return limit, cursor
def page(items, limit, scope):
more = len(items) > limit
items = items[:limit]
cursor = None
if more:
last = items[-1]
cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=')
return {'items': items, 'next_cursor': cursor}
+13
View File
@@ -0,0 +1,13 @@
"""One-shot migration job; replicas never execute DDL."""
from pathlib import Path
import psycopg
from postgres import setting
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=5) as conn:
conn.execute('SELECT pg_advisory_xact_lock(2301003)')
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)')
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
if not conn.execute('SELECT version FROM schema_migrations WHERE 1=1 AND version = %s', (path.name,)).fetchone():
conn.execute(path.read_text())
conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.name,))
print('Applied', path.name)
+158
View File
@@ -0,0 +1,158 @@
"""Bounded S3 operations. No root credentials or filename-derived object keys."""
import contextlib
import telemetry as tele
import hashlib
import os
import tempfile
import threading
import time
import unicodedata
import warnings
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, PngImagePlugin, UnidentifiedImageError
from common import APIError, invalid
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024
PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
UPLOAD_SLOTS = threading.BoundedSemaphore(8)
DECODE_SLOTS = threading.BoundedSemaphore(1)
def unavailable():
raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно')
def filename(value):
if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip():
invalid()
if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value):
invalid()
try:
value.encode('utf-8')
except UnicodeError:
invalid()
return value
@contextlib.contextmanager
def bounded_upload():
if not UPLOAD_SLOTS.acquire(blocking=False):
raise APIError(503, 'unavailable', 'Все слоты загрузки заняты')
try:
yield
finally:
UPLOAD_SLOTS.release()
@contextlib.contextmanager
def receive(handler):
lengths = handler.headers.get_all('Content-Length', [])
if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'):
invalid()
try:
length = int(lengths[0])
except ValueError:
invalid()
if length > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'Максимум 10 MiB')
if length <= 0:
invalid()
mime = handler.headers.get_content_type()
if mime not in ('image/png', 'image/jpeg'):
raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG')
deadline = time.monotonic() + 15
digest = hashlib.sha256()
# Ephemeral bounded spool is never a durable source of a confirmed attachment.
with tempfile.TemporaryFile(mode='w+b') as source:
remaining = length
while remaining:
handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic())))
if time.monotonic() >= deadline:
invalid()
try:
chunk = handler.rfile.read1(min(64 * 1024, remaining))
except (OSError, TimeoutError):
invalid()
if not chunk:
invalid()
digest.update(chunk)
source.write(chunk)
remaining -= len(chunk)
if not DECODE_SLOTS.acquire(timeout=10):
raise APIError(503, 'unavailable', 'Проверка изображений перегружена')
try:
source.flush()
for decode in (False,):
source.seek(0)
# Pillow.close releases pixel buffers and may close its input file.
# A duplicate descriptor preserves the original for the S3 PUT.
with os.fdopen(os.dup(source.fileno()), 'rb') as image_file:
image = Image.open(image_file, formats=['PNG', 'JPEG'])
try:
if image.width * image.height > MAX_PIXELS:
raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей')
if Image.MIME[image.format] != mime:
raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению')
if decode:
image.load()
else:
image.verify()
finally:
image.close() # Free pixels BEFORE releasing the decoder semaphore.
except (Image.DecompressionBombWarning, Image.DecompressionBombError):
raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None
finally:
DECODE_SLOTS.release()
source.seek(0)
yield source, length, mime, digest.hexdigest()
class Objects:
def __init__(self, setting):
self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments')
self.client = boto3.client('s3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:8333'), region_name='us-east-1', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required'))
self.client = tele.S3Client(self.client)
def head(self, key):
try:
return self.client.head_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'):
return None
unavailable()
except BotoCoreError:
unavailable()
def ready(self):
try:
self.client.head_bucket(Bucket=self.bucket)
except (ClientError, BotoCoreError):
unavailable()
def put(self, key, source, length, mime, digest):
try:
self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest})
except (ClientError, BotoCoreError):
unavailable()
def get(self, key):
try:
return self.client.get_object(Bucket=self.bucket, Key=key)
except (ClientError, BotoCoreError):
unavailable()
def delete(self, key):
try:
self.client.delete_object(Bucket=self.bucket, Key=key)
except (ClientError, BotoCoreError):
unavailable()
+393
View File
@@ -0,0 +1,393 @@
"""Durable outbox, Redis Streams delivery, bounded worker and fenced result commits."""
import argparse
import concurrent.futures
import contextlib
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import random
import re
import signal
import subprocess
import sys
import tempfile
import threading
import time
import uuid
import telemetry as tele
import psycopg
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
import redis
from botocore.exceptions import ClientError, BotoCoreError
from common import APIError
from objects import Objects, MAX_BYTES
from postgres import setting
STREAM, GROUP = 'attachment-jobs', 'image-workers'
LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '60'))
STOP = threading.Event()
def raw_connect():
return psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row, autocommit=True, connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1500')
@contextlib.contextmanager
def connect():
with tele.span('db.transaction', {'db.system':'postgresql'}, tele.SpanKind.CLIENT):
with raw_connect() as connection:
yield connection
def broker():
client = redis.Redis.from_url(setting('BROKER_URL'), decode_responses=True, socket_connect_timeout=1, socket_timeout=3)
try:
client.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
except redis.ResponseError as error:
if 'BUSYGROUP' not in str(error):
raise
return client
def event(job):
return {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], 'attempt': job['attempt'] + 1, 'created_at': job['created_at'].isoformat().replace('+00:00', 'Z'), **(job.get('trace_context') or {})}
def validate(message):
required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
if not isinstance(message, dict) or set(message) - required - {'traceparent', 'tracestate'} or not required <= set(message):
return False
try:
uuid.UUID(message['job_id']); uuid.UUID(message['attachment_id'])
stamp = dt.datetime.fromisoformat(message['created_at'].replace('Z', '+00:00'))
if stamp.tzinfo is None or not message['created_at'].endswith('Z'):
return False
if 'traceparent' in message and (not isinstance(message['traceparent'], str) or not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', message['traceparent'])):
return False
if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512):
return False
return type(message['schema_version']) is int and message['schema_version'] == 1 and message['type'] == 'attachment.process.v1' and all(type(message[k]) is int and message[k] >= 1 for k in ['pipeline_version', 'attempt'])
except (ValueError, TypeError, AttributeError):
return False
def audit(conn, job, name):
conn.execute('INSERT INTO job_events (job_id, event, attempt) VALUES (%s, %s, %s)', (job['id'], name, job['attempt']))
def retry(conn, job, code, permanent=False):
failed = permanent or job['attempt'] >= job['max_attempts']
state = 'failed' if failed else 'queued'
code = code if permanent or not failed else 'retry_exhausted'
delay = min(8, .5 * 2 ** min(job['attempt'] - 1, 4)) + random.uniform(0, .5)
conn.execute('''
UPDATE attachment_jobs
SET state = %s
, error_code = %s
, lease_token = NULL
, lease_until = NULL
, published_at = NULL
, next_attempt_at = clock_timestamp() + %s * INTERVAL '1 second'
, updated_at = clock_timestamp()
WHERE 1=1
AND id = %s
''', (state, code, delay, job['id']))
conn.execute('''
UPDATE attachments
SET state = %s
, error_code = %s
, updated_at = clock_timestamp()
WHERE 1=1
AND current_job_id = %s
''', (state, code if failed else None, job['id']))
audit(conn, job, 'failed:' + code if failed else 'retry:' + code)
tele.count('job_retries', {'job_type':'image', 'outcome':'failed' if failed else 'retry'})
tele.log('job_failed' if failed else 'job_retry', job_id=str(job['id']), attempt=job['attempt'], error_code=code)
def dispatch_once(client, fault=None):
with connect() as conn:
with conn.transaction():
expired = conn.execute("SELECT * FROM attachment_jobs WHERE 1=1 AND state = 'processing' AND lease_until < clock_timestamp() FOR UPDATE SKIP LOCKED LIMIT 20").fetchall()
for job in expired:
retry(conn, job, 'lease_expired')
with conn.transaction():
jobs = conn.execute('''
SELECT j.*
FROM attachment_jobs j
JOIN attachments a ON a.current_job_id = j.id
WHERE 1=1
AND j.state = 'queued'
AND j.next_attempt_at <= clock_timestamp()
AND (j.published_at IS NULL OR j.published_at < clock_timestamp() - %s * INTERVAL '1 second')
ORDER BY j.created_at
LIMIT 10
FOR UPDATE OF j SKIP LOCKED
''', (LEASE,)).fetchall()
for job in jobs:
with tele.span('attachment publish', {'messaging.system':'redis', 'messaging.destination.name':STREAM},
tele.SpanKind.PRODUCER, job.get('trace_context') or {}):
message = event(job)
message.update(tele.carrier())
client.xadd(STREAM, {'event': json.dumps(message, separators=(',', ':'))})
tele.log('job_published', job_id=str(job['id']), attempt=job['attempt']+1)
if fault:
fault('after_publish', job)
conn.execute('UPDATE attachment_jobs SET published_at = clock_timestamp() WHERE 1=1 AND id = %s', (job['id'],))
audit(conn, job, 'published')
return len(jobs)
def claim(message):
with connect() as conn, conn.transaction():
job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (message['job_id'],)).fetchone()
if job is None:
return None
attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (job['attachment_id'],)).fetchone()
if str(job['attachment_id']) != message['attachment_id'] or job['pipeline_version'] != message['pipeline_version']:
return None
if attachment['current_job_id'] != job['id'] or attachment['pipeline_version'] != job['pipeline_version']:
audit(conn, job, 'stale_noop')
return None
if job['state'] in ('ready', 'failed', 'superseded'):
audit(conn, job, 'terminal_noop')
return None
if job['state'] != 'queued' or job['next_attempt_at'] > dt.datetime.now(dt.timezone.utc) or message['attempt'] != job['attempt'] + 1:
audit(conn, job, 'duplicate_or_early_noop')
return None
token = uuid.uuid4()
job = conn.execute('''
UPDATE attachment_jobs
SET state = 'processing'
, attempt = attempt + 1
, lease_token = %s
, lease_until = clock_timestamp() + %s * INTERVAL '1 second'
, updated_at = clock_timestamp()
WHERE 1=1
AND id = %s
RETURNING *
''', (token, LEASE, job['id'])).fetchone()
conn.execute("UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job['id'],))
audit(conn, job, 'claimed')
return job, attachment
class Permanent(Exception):
pass
def immutable(objects, key, path, mime, sha):
head = objects.head(key)
if head is not None:
if head['ContentLength'] != path.stat().st_size or head.get('Metadata', {}).get('sha256') != sha:
raise Permanent('result_conflict')
return
try:
with path.open('rb') as source:
objects.client.put_object(Bucket=objects.bucket, Key=key, Body=source, ContentLength=path.stat().st_size, ContentType=mime, Metadata={'sha256': sha}, IfNoneMatch='*')
except ClientError as error:
if error.response['ResponseMetadata']['HTTPStatusCode'] in (409, 412):
head = objects.head(key)
if head and head['ContentLength'] == path.stat().st_size and head.get('Metadata', {}).get('sha256') == sha:
return
raise
def process_core(message, fault=None):
claimed = claim(message)
if claimed is None:
return 'noop'
job, attachment = claimed
objects = Objects(setting)
try:
if fault:
fault('before_side_effect', job)
with tempfile.TemporaryDirectory(prefix='job-') as temporary:
directory = Path(temporary)
source = directory / 'original'
head = objects.head(job['source_key'])
if head is None:
raise Permanent('source_missing')
response = objects.get(job['source_key'])
digest, size = hashlib.sha256(), 0
deadline = time.monotonic() + 15
with response['Body'] as stream, source.open('wb') as out:
for chunk in iter(lambda: stream.read(64 * 1024), b''):
size += len(chunk)
if size > MAX_BYTES or time.monotonic() > deadline:
raise Permanent('invalid_image')
digest.update(chunk); out.write(chunk)
if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']:
raise Permanent('checksum_mismatch')
result = subprocess.run([sys.executable, str(Path(__file__).with_name('render.py')), str(source), str(directory), str(job['pipeline_version'])], capture_output=True, timeout=int(os.environ.get('JOB_TIMEOUT_SECONDS', '20')))
if result.returncode == 2:
raise Permanent('invalid_image')
if result.returncode:
raise RuntimeError('render_failed')
variants = json.loads(result.stdout)
prefix = 'results/' + str(attachment['id']) + '/v' + str(job['pipeline_version']) + '/'
original_key = prefix + 'original'
immutable(objects, original_key, source, attachment['content_type'], attachment['sha256'])
if fault:
fault('after_original', job)
for variant in variants:
variant['object_key'] = prefix + variant['name']
immutable(objects, variant['object_key'], directory / variant['name'], variant['content_type'], variant['sha256'])
if fault:
fault('after_objects', job)
with connect() as conn, conn.transaction():
current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone()
changed = conn.execute('''
UPDATE attachments
SET state = 'ready'
, object_key = %s
, variants = %s
, error_code = NULL
, updated_at = clock_timestamp()
WHERE 1=1
AND current_job_id = %s
AND pipeline_version = %s
AND %s
RETURNING id
''', (original_key, Jsonb(variants), job['id'], job['pipeline_version'], current['state'] == 'processing' and current['lease_token'] == job['lease_token'] and current['lease_until'] > dt.datetime.now(dt.timezone.utc))).fetchone()
if changed:
conn.execute("UPDATE attachment_jobs SET state = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE 1=1 AND id = %s", (job['id'],))
audit(conn, job, 'ready')
else:
audit(conn, job, 'fenced_noop')
if fault:
fault('after_commit', job)
if changed and job['source_key'].startswith('staging/'):
objects.delete(job['source_key'])
if changed:
tele.duration('job_ready_duration', (dt.datetime.now(dt.timezone.utc)-job['created_at']).total_seconds(), {'job_type':'image'})
return 'ready' if changed else 'fenced'
except Exception as error:
code = str(error) if isinstance(error, Permanent) else 'processing_timeout' if isinstance(error, subprocess.TimeoutExpired) else 'storage_unavailable' if isinstance(error, (APIError, ClientError, BotoCoreError)) else 'processing_failed'
with connect() as conn, conn.transaction():
current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone()
if current['state'] == 'processing' and current['lease_token'] == job['lease_token']:
retry(conn, current, code, isinstance(error, Permanent))
return 'failed_or_retry'
finally:
objects.client.close()
def process(message, fault=None):
started = time.monotonic()
with tele.span('attachment process', {'messaging.system':'redis', 'messaging.destination.name':STREAM,
'job.id':message['job_id'], 'job.attempt':message['attempt']},
tele.SpanKind.CONSUMER, message):
with tele.LOCK:
tele.STATE['worker_busy'] += 1
outcome = 'error'
try:
outcome = process_core(message, fault)
if outcome == 'failed_or_retry':
tele.trace.get_current_span().set_status(tele.Status(tele.StatusCode.ERROR))
return outcome
finally:
with tele.LOCK:
tele.STATE['worker_busy'] -= 1
attributes = {'job_type':'image', 'outcome':outcome}
tele.count('jobs_processed', attributes)
tele.duration('job_processing_duration', time.monotonic()-started, attributes)
tele.log('job_processed', job_id=message['job_id'], attempt=message['attempt'], outcome=outcome)
def observe_queue():
try:
with raw_connect() as conn:
row = conn.execute("""
SELECT
count(*) AS depth
, coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS age
FROM attachment_jobs
WHERE 1=1
AND state IN ('queued', 'processing')
""").fetchone()
tele.STATE.update(queue_depth=row['depth'], queue_oldest_age=float(row['age']), queue_observation_ok=1)
except psycopg.Error:
tele.STATE['queue_observation_ok'] = 0
def ack(client, identifier):
# Atomic ack+delete; PostgreSQL is the durable completion/replay registry.
client.eval("redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])", 1, STREAM, GROUP, identifier)
def consume(once=False, fault=None):
client = broker()
consumer = uuid.uuid4().hex
while not STOP.is_set():
messages = []
try:
pending = client.xautoclaim(STREAM, GROUP, consumer, min_idle_time=LEASE * 1000, start_id='0-0', count=1)[1]
if pending:
messages = pending
else:
batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=500)
messages = batches[0][1] if batches else []
for identifier, fields in messages:
raw = fields.get('event', '')
try:
message = json.loads(raw) if len(raw) <= 4096 else None
except ValueError:
message = None
if validate(message):
process(message, fault)
else:
client.xadd('rejected-jobs', {'source_message_id': identifier, 'sha256': hashlib.sha256(raw.encode()).hexdigest(), 'reason': 'invalid_envelope'}, maxlen=1000, approximate=False)
ack(client, identifier)
if once:
return
except (redis.RedisError, psycopg.Error):
STOP.wait(.5)
if once and not messages:
return
client.close()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('mode', choices=['dispatch', 'work'])
parser.add_argument('--once', action='store_true')
parser.add_argument('--fault-point', choices=['before_side_effect', 'after_original', 'after_objects', 'after_commit', 'after_publish'])
args = parser.parse_args()
tele.configure('messenger-dispatcher' if args.mode=='dispatch' else 'messenger-worker')
# Operator-only opt-in injection; no request header or public endpoint controls it.
def fault(point, job):
if point == args.fault_point:
os._exit(77)
signal.signal(signal.SIGTERM, lambda *_: STOP.set())
signal.signal(signal.SIGINT, lambda *_: STOP.set())
if args.mode == 'dispatch':
client = broker()
while not STOP.is_set():
observe_queue()
try:
dispatch_once(client, fault)
except (redis.RedisError, psycopg.Error):
pass
if args.once:
break
STOP.wait(.2)
else:
count = int(os.environ.get('WORKER_CONCURRENCY', '1'))
if not 1 <= count <= 2:
raise ValueError('WORKER_CONCURRENCY must be 1..2')
with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool:
futures = [pool.submit(consume, args.once, fault) for _ in range(count)]
for result in futures:
result.result()
if __name__ == '__main__':
try:
main()
finally:
tele.shutdown()
+414
View File
@@ -0,0 +1,414 @@
import base64
import contextlib
import datetime as dt
import hashlib
import hmac
import json
import os
import re
from pathlib import Path
import secrets
import threading
import uuid
import telemetry as tele
import psycopg
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from psycopg_pool import ConnectionPool, PoolTimeout
import redis
from objects import Objects, unavailable
from common import APIError, fields, identifier, invalid, missing, page, text, unauthorized, username
KDF_SLOTS = threading.BoundedSemaphore(2)
def setting(name):
path = os.environ.get(name + '_FILE')
return Path(path).read_text().strip() if path else os.environ[name]
def public(row):
return {k: (v.decode('utf-8') if isinstance(v, bytes) else [str(x) for x in v] if isinstance(v, list) else str(v) if isinstance(v, uuid.UUID) else v.isoformat(timespec='microseconds').replace('+00:00', 'Z') if isinstance(v, dt.datetime) else v) for k, v in row.items() if k != 'password_hash'}
def password_hash(password, salt=None):
salt = salt or secrets.token_bytes(16)
if not KDF_SLOTS.acquire(timeout=1.5):
raise APIError(503, 'unavailable', 'Аутентификация временно перегружена')
try:
digest = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024)
finally:
KDF_SLOTS.release()
return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex()
class Store:
persistent = True
def __init__(self):
self.ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600'))
if not 1 <= self.ttl <= 86400:
raise ValueError('SESSION_TTL_SECONDS must be 1..86400')
self.access_mode = os.environ.get('DB_ACCESS_MODE', 'optimized')
if self.access_mode not in ('baseline', 'optimized'):
raise ValueError('DB_ACCESS_MODE must be baseline or optimized')
pool_size = int(os.environ.get('DB_POOL_MAX', '8'))
if not 1 <= pool_size <= 32:
raise ValueError('DB_POOL_MAX must be 1..32')
self.pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=pool_size, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=2000 -c lock_timeout=1500'}, open=True)
self.pool.wait(timeout=10)
tele.pool(self.pool)
self.cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.15, socket_timeout=0.15, decode_responses=True)
self.objects = Objects(setting)
self.dummy_hash = password_hash('dummy-password-for-unknown-users')
with self.connection() as conn:
if not conn.execute("SELECT version FROM schema_migrations WHERE 1=1 AND version = '006_observability.sql'").fetchone():
raise RuntimeError('Run the separate migration service first')
@contextlib.contextmanager
def connection(self):
try:
with tele.span('db.transaction', {'db.system':'postgresql'}, tele.SpanKind.CLIENT):
with self.pool.connection() as conn:
yield conn
except psycopg.errors.UniqueViolation:
raise APIError(409, 'conflict', 'Имя пользователя занято') from None
except (psycopg.Error, PoolTimeout):
raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None
def close(self):
self.pool.close()
self.cache.close()
self.objects.client.close()
def ready(self):
self.objects.ready()
with self.connection() as conn:
conn.execute('SELECT 1')
def register(self, body):
fields(body, ('username', 'display_name', 'password'))
name, display = username(body['username']), text(body['display_name'], 100)
# Passwords may contain spaces and colons; only length/UTF-8 are restricted.
password = body['password']
if not isinstance(password, str) or not 12 <= len(password) <= 128:
invalid()
try:
password.encode('utf-8')
except UnicodeError:
invalid()
hashed = password_hash(password)
with self.connection() as conn:
row = conn.execute('''
INSERT INTO users (id, username, display_name, password_hash)
VALUES (%s, %s, %s, %s)
RETURNING id, username, display_name, created_at
''', (uuid.uuid4(), name, display.encode('utf-8'), hashed)).fetchone()
return public(row)
def login(self, headers):
authorization = headers.get('Authorization', '')
try:
scheme, encoded = authorization.split(' ', 1)
if scheme.lower() != 'basic':
unauthorized()
name, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
except (ValueError, UnicodeError):
unauthorized()
if not re_username(name) or not 12 <= len(password) <= 128:
unauthorized()
with self.connection() as conn:
row = conn.execute('SELECT id, username, display_name, created_at, password_hash FROM users WHERE 1=1 AND username = %s', (name,)).fetchone()
hashed = row['password_hash'] if row else self.dummy_hash
verified = hmac.compare_digest(password_hash(password, bytes.fromhex(hashed.split('$')[4])), hashed)
if row is None or not verified:
unauthorized()
token = secrets.token_urlsafe(32)
with self.connection() as conn:
expiry = conn.execute('''
INSERT INTO sessions (token_hash, user_id, expires_at)
VALUES (%s, %s, clock_timestamp() + %s * INTERVAL '1 second')
RETURNING expires_at
''', (hashlib.sha256(token.encode()).hexdigest(), row['id'], self.ttl)).fetchone()['expires_at']
return {'token': token, 'token_type': 'Bearer', 'expires_at': public({'expires_at': expiry})['expires_at'], 'user': public(row)}
def token_hash(self, headers):
parts = headers.get('Authorization', '').split(' ')
if len(parts) != 2 or parts[0].lower() != 'bearer' or len(parts[1]) != 43 or not parts[1].isascii():
unauthorized()
return hashlib.sha256(parts[1].encode()).hexdigest()
def authenticate(self, headers):
token_hash = self.token_hash(headers)
with self.connection() as conn:
session = conn.execute('''
SELECT user_id
FROM sessions
WHERE 1=1
AND token_hash = %s
AND expires_at > clock_timestamp()
''', (token_hash,)).fetchone()
if session is None:
unauthorized()
key = 'user:' + str(session['user_id'])
try:
cached = self.cache.get(key)
if cached:
return json.loads(cached)
except (redis.RedisError, ValueError):
pass
row = conn.execute('SELECT id, username, display_name, created_at FROM users WHERE 1=1 AND id = %s', (session['user_id'],)).fetchone()
if row is None:
unauthorized()
user = public(row)
try:
self.cache.setex(key, 60, json.dumps(user))
except redis.RedisError:
pass
return user
def logout(self, headers):
with self.connection() as conn:
conn.execute('DELETE FROM sessions WHERE 1=1 AND token_hash = %s', (self.token_hash(headers),))
def chat_in(self, conn, actor, chat_id):
row = conn.execute('''
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE 1=1
AND c.id = %s
AND cm.user_id = %s
''', (chat_id, actor['id'])).fetchone()
if row is None:
missing()
members = conn.execute('SELECT user_id FROM chat_members WHERE 1=1 AND chat_id = %s ORDER BY user_id', (chat_id,)).fetchall()
return dict(public(row), member_ids=[str(v['user_id']) for v in members])
def chat(self, actor, chat_id):
with self.connection() as conn:
return self.chat_in(conn, actor, chat_id)
def create_chat(self, actor, body):
fields(body, ('title', 'member_ids'))
title = text(body['title'], 100)
members = body['member_ids']
if not isinstance(members, list) or not 1 <= len(members) <= 100:
invalid()
members = [identifier(v) for v in members]
if len(set(members)) != len(members) or actor['id'] not in members:
invalid()
with self.connection() as conn:
chat_id = uuid.uuid4()
conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat_id, title.encode('utf-8')))
# Checking after INSERT exercises rollback of the whole composite operation.
found = conn.execute('SELECT id FROM users WHERE 1=1 AND id = ANY(%s::uuid[])', (members,)).fetchall()
if len(found) != len(members):
invalid()
with conn.cursor() as cur:
cur.executemany('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', [(chat_id, v) for v in members])
return self.chat_in(conn, actor, chat_id)
def list_chats(self, actor, limit, cursor, scope):
with self.connection() as conn:
where = 'AND (c.created_at, c.id) > (%s::timestamptz, %s::uuid)' if cursor else ''
params = [actor['id']] + list(cursor or ()) + [limit + 1]
rows = conn.execute('''
SELECT c.id
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE 1=1
AND cm.user_id = %s
''' + where + '''
ORDER BY c.created_at, c.id
LIMIT %s
''', params).fetchall()
return page([self.chat_in(conn, actor, v['id']) for v in rows], limit, scope)
def require_member(self, conn, actor, chat_id):
if self.access_mode == 'baseline':
return self.chat_in(conn, actor, chat_id)
if not conn.execute('''
SELECT 1
FROM chat_members
WHERE 1=1
AND chat_id = %s
AND user_id = %s
''', (chat_id, actor['id'])).fetchone():
missing()
def send(self, actor, chat_id, body, key):
if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
invalid()
if not isinstance(body, dict) or set(body) - {'text', 'attachment_ids'} or 'text' not in body:
invalid()
attachments = body.get('attachment_ids', [])
if not isinstance(attachments, list) or len(attachments) > 10:
invalid()
attachments = [identifier(v) for v in attachments]
if len(set(attachments)) != len(attachments):
invalid()
content = body['text']
if not isinstance(content, str) or len(content) > 4000 or (not attachments and not content.strip()):
invalid()
try:
content.encode('utf-8')
except UnicodeError:
invalid()
canonical = json.dumps({'text': content, 'attachment_ids': attachments}, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
digest = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
with self.connection() as conn:
self.require_member(conn, actor, chat_id)
# The unique key arbitrates concurrent processes; everything commits together.
claimed = conn.execute('''
INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, expires_at)
VALUES (%s, %s, %s, %s, clock_timestamp() + INTERVAL '24 hours')
ON CONFLICT (user_id, chat_id, key) DO UPDATE
SET request_hash = EXCLUDED.request_hash
, message_id = NULL
, expires_at = EXCLUDED.expires_at
WHERE 1=1
AND idempotency_keys.expires_at <= clock_timestamp()
RETURNING request_hash
''', (actor['id'], chat_id, key, digest)).fetchone()
if claimed is None:
previous = conn.execute('''
SELECT ik.request_hash
, m.id
, m.chat_id
, m.sender_id
, m.text
, m.created_at
, m.attachment_ids
FROM idempotency_keys ik
JOIN messages m ON m.id = ik.message_id
WHERE 1=1
AND ik.user_id = %s
AND ik.chat_id = %s
AND ik.key = %s
''', (actor['id'], chat_id, key)).fetchone()
if previous['request_hash'] != digest:
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
previous.pop('request_hash')
return public(previous)
if attachments:
rows = conn.execute('''
SELECT id, state
FROM attachments
WHERE 1=1
AND id = ANY(%s::uuid[])
AND chat_id = %s
ORDER BY id
FOR SHARE
''', (attachments, chat_id)).fetchall()
if len(rows) != len(attachments):
missing()
if any(v['state'] not in ('ready', 'queued', 'processing') for v in rows):
raise APIError(409, 'conflict', 'Вложение ещё не готово')
# Per-chat row lock preserves committed history order across replicas/clocks.
stamp = conn.execute('''
UPDATE chats
SET last_message_at = GREATEST(clock_timestamp(), last_message_at + INTERVAL '1 microsecond')
WHERE 1=1
AND id = %s
RETURNING last_message_at
''', (chat_id,)).fetchone()['last_message_at']
row = conn.execute('''
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
VALUES (%s, %s, %s, %s, %s, %s::uuid[])
RETURNING id, chat_id, sender_id, text, created_at, attachment_ids
''', (uuid.uuid4(), chat_id, actor['id'], content.encode('utf-8'), stamp, attachments)).fetchone()
if attachments:
with conn.cursor() as cur:
cur.executemany('INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)', [(row['id'], v) for v in attachments])
conn.execute('''
UPDATE idempotency_keys
SET message_id = %s
WHERE 1=1
AND user_id = %s
AND chat_id = %s
AND key = %s
''', (row['id'], actor['id'], chat_id, key))
return public(row)
def history(self, actor, chat_id, limit, cursor, scope):
with self.connection() as conn:
self.require_member(conn, actor, chat_id)
where = 'AND (created_at, id) > (%s::timestamptz, %s::uuid)' if cursor else ''
params = [chat_id] + list(cursor or ()) + [limit + 1]
rows = conn.execute('''
SELECT id, chat_id, sender_id, text, created_at, attachment_ids
FROM messages
WHERE 1=1
AND chat_id = %s
''' + where + '''
ORDER BY created_at, id
LIMIT %s
''', params).fetchall()
return page([public(v) for v in rows], limit, scope)
def upload(self, actor, chat_id, name, source, length, mime, digest):
attachment_id, job_id = uuid.uuid4(), uuid.uuid4()
key = 'staging/' + str(attachment_id)
with self.connection() as conn:
self.require_member(conn, actor, chat_id)
conn.execute('SELECT pg_advisory_xact_lock(2301005)')
pending = conn.execute("SELECT count(*) AS n FROM attachments WHERE 1=1 AND state IN ('uploading', 'queued', 'processing')").fetchone()['n']
if pending >= int(os.environ.get('MAX_PENDING_JOBS', '100')):
raise APIError(503, 'unavailable', 'Очередь принятых работ заполнена')
conn.execute('''
INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256, state)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')
''', (attachment_id, chat_id, actor['id'], name, 'results/' + str(attachment_id) + '/v1/original', key, mime, length, digest))
with self.connection() as conn:
if conn.execute('SELECT id FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() is None:
unavailable()
self.objects.put(key, source, length, mime, digest)
conn.execute('''
INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts, trace_context)
VALUES (%s, %s, 1, %s, %s, %s)
''', (job_id, attachment_id, key, int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), Jsonb(tele.carrier())))
row = conn.execute('''
UPDATE attachments
SET state = 'queued'
, current_job_id = %s
, updated_at = clock_timestamp()
WHERE 1=1
AND id = %s
RETURNING *
''', (job_id, attachment_id)).fetchone()
tele.log('attachment_accepted', attachment_id=str(attachment_id), job_id=str(job_id))
# Capture the queued snapshot, even if the worker finishes before HTTP reply.
return self.attachment_public(row)
def attachment_public(self, row):
keys = ('id', 'chat_id', 'owner_id', 'filename', 'content_type', 'size_bytes', 'sha256', 'created_at', 'updated_at')
result = public({k: row[k] for k in keys})
variants = [{k: v for k, v in item.items() if k != 'object_key'} for item in row['variants']]
return dict(result, status=row['state'], variants=variants, error_code=row['error_code'])
def attachment(self, actor, attachment_id):
with self.connection() as conn:
row = conn.execute('''
SELECT a.*
FROM attachments a
JOIN chat_members cm ON cm.chat_id = a.chat_id
WHERE 1=1
AND a.id = %s
AND cm.user_id = %s
''', (attachment_id, actor['id'])).fetchone()
if row is None:
missing()
if row['state'] == 'storage_missing':
unavailable()
if row['state'] == 'uploading':
raise APIError(409, 'conflict', 'Приём не завершён')
return row
def re_username(value):
import re
return re.fullmatch(r'[a-z0-9_]{3,32}', value)
+64
View File
@@ -0,0 +1,64 @@
"""Isolated image decoder/resizer; no network, database or credentials."""
import hashlib
import json
import math
from pathlib import Path
import resource
import sys
import warnings
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
resource.setrlimit(resource.RLIMIT_CPU, (15, 16))
from PIL import Image, ImageOps, PngImagePlugin
Image.MAX_IMAGE_PIXELS = 40_000_000
PngImagePlugin.MAX_TEXT_CHUNK = PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
def render(source, directory, version):
output = []
with Image.open(source, formats=['PNG', 'JPEG']) as image:
if image.width * image.height > 40_000_000:
raise ValueError('pixels')
image.load()
ImageOps.exif_transpose(image, in_place=True)
rgba = None
if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info):
rgba = image if image.mode == 'RGBA' else image.convert('RGBA')
rgb = Image.new('RGB', rgba.size, 'white')
alpha = rgba.getchannel('A')
rgb.paste(rgba, mask=alpha)
alpha.close()
else:
rgb = image if image.mode == 'RGB' else image.convert('RGB')
try:
for name, bound, fmt, quality in [('thumbnail', 256, 'JPEG', 85), ('crop', 128, 'JPEG', 85), ('optimized', 1280, 'WEBP', 82 if version == 1 else 70)]:
if name == 'crop':
result = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(.5, .5))
else:
scale = min(1, bound / max(rgb.size))
size = tuple(max(1, math.floor(v * scale)) for v in rgb.size)
result = rgb.resize(size, Image.Resampling.LANCZOS)
try:
path = directory / name
options = {'quality': quality, 'exif': b'', 'icc_profile': None}
if fmt == 'WEBP':
options['method'] = 4
result.save(path, format=fmt, **options)
raw = path.read_bytes()
output.append({'name': name, 'width': result.width, 'height': result.height, 'content_type': 'image/jpeg' if fmt == 'JPEG' else 'image/webp', 'size_bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()})
finally:
result.close()
finally:
if rgb is not image:
rgb.close()
if rgba is not None and rgba is not image:
rgba.close()
return output
if __name__ == '__main__':
try:
print(json.dumps(render(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3]))))
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError, Image.DecompressionBombWarning):
raise SystemExit(2)
+317
View File
@@ -0,0 +1,317 @@
"""Bounded threaded API; persistent correctness state is shared across replicas."""
import concurrent.futures
import contextlib
import http.server
import json
import logging
import os
import time
import signal
import socket
import threading
import uuid
from urllib.parse import parse_qs, urlsplit
from objects import bounded_upload, receive, filename, unavailable
import telemetry as tele
from common import APIError, identifier, invalid, missing, page_args
LOG = logging.getLogger('messenger')
def unique_object(pairs):
result = {}
for key, value in pairs:
if key in result:
invalid()
result[key] = value
return result
class Handler(http.server.BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
def setup(self):
super().setup()
self.connection.settimeout(3)
def send_error(self, code, message=None, explain=None):
# BaseHTTPRequestHandler otherwise emits HTML without contract headers.
self.request_id = uuid.uuid4().hex
self.route = ''
self.close_connection = True
status = 413 if code == 413 else 400
self.reply(status, {'error': {'code': 'payload_too_large' if status == 413 else 'invalid_request', 'message': 'Некорректный HTTP-запрос', 'request_id': self.request_id}})
def log_message(self, *args):
pass
def reply(self, status, body):
if status == 200 and isinstance(body, tuple):
return self.download(*body)
raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8')
if status >= 400:
self.close_connection = True
self.send_response(status)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(raw)))
if self.close_connection:
self.send_header('Connection', 'close')
self.send_header('X-Request-Id', self.request_id)
self.send_header('X-Instance-Id', self.server.instance_id)
if status == 401 and self.server.store.persistent:
self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
self.end_headers()
if self.command != 'HEAD':
self.wfile.write(raw)
def download(self, row, response, chunk):
stream = response['Body']
try:
self.send_response(200)
self.send_header('Content-Type', row['content_type'])
self.send_header('Content-Length', str(row['size_bytes']))
self.send_header('X-Request-Id', self.request_id)
self.send_header('X-Instance-Id', self.server.instance_id)
self.send_header('X-Content-Type-Options', 'nosniff')
self.send_header('Cache-Control', 'private, no-store')
self.end_headers()
deadline = time.monotonic() + 15
while chunk:
self.wfile.write(chunk)
if time.monotonic() > deadline:
raise TimeoutError()
chunk = stream.read(64 * 1024)
finally:
stream.close()
def body(self):
lengths = self.headers.get_all('Content-Length', [])
if len(lengths) != 1 or self.headers.get('Transfer-Encoding'):
invalid()
try:
length = int(lengths[0])
except ValueError:
invalid()
if length > 32768:
raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт')
if length <= 0:
invalid()
if self.headers.get_content_type() != 'application/json':
raise APIError(415, 'unsupported_media_type', 'Требуется application/json')
try:
raw = self.rfile.read(length)
if len(raw) != length:
invalid()
return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid())
except (ValueError, UnicodeError, RecursionError):
invalid()
def dispatch(self):
self.request_id = uuid.uuid4().hex
self.route = urlsplit(self.path).path
template = tele.route(self.route)
method = self.command if self.command in ('GET','POST','DELETE','PUT','PATCH','HEAD','OPTIONS') else 'OTHER'
attrs = {'method':method, 'route':template}
started = time.monotonic()
context = tele.span(method+' '+template, {'http.request.method':method, 'http.route':template},
tele.SpanKind.SERVER, {k:self.headers[k] for k in ('traceparent','tracestate') if k in self.headers})
# Health probes do not enter business telemetry or create noisy dependency spans.
if template.startswith('/health/'):
context = tele.suppressed()
status = 500
with context as current:
try:
status, result = self.route_request()
except APIError as exc:
status = exc.status
result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}}
except Exception:
result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}}
try:
self.reply(status, result)
except (BrokenPipeError, ConnectionResetError, TimeoutError, OSError):
self.close_connection = True
if not template.startswith('/health/'):
attrs['status_class'] = str(status//100)+'xx'
tele.count('http_server_requests', attrs)
tele.duration('http_server_duration', time.monotonic()-started, attrs)
if current is not None:
current.set_attribute('http.response.status_code', status)
if status >= 500:
current.set_status(tele.Status(tele.StatusCode.ERROR))
tele.log('http_request', request_id=self.request_id, method=method, route=template, status=status, duration_seconds=round(time.monotonic()-started,6))
def route_request(self):
method, path, store = self.command, self.route, self.server.store
if method == 'GET' and path in ('/health/live', '/health/ready'):
if path.endswith('ready'):
store.ready()
return 200, {'status': 'ok'}
if method == 'POST' and path == '/api/v1/users':
return 201, store.register(self.body())
if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions':
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
invalid()
return 201, store.login(self.headers)
actor = store.authenticate(self.headers)
if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current':
store.logout(self.headers)
return 204, None
if method == 'GET' and path == '/api/v1/users/me':
return 200, actor
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
if path == '/api/v1/chats':
if method == 'POST':
return 201, store.create_chat(actor, self.body())
if method == 'GET':
scope = 'chats:' + actor['id']
limit, cursor = page_args(query, scope)
return 200, store.list_chats(actor, limit, cursor, scope)
parts = path.split('/')
if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'attachments'] and method == 'GET':
row = store.attachment(actor, identifier(parts[4]))
if len(parts) == 5:
if query:
invalid()
return 200, store.attachment_public(row)
if parts[5] == 'content':
if set(query) - {'variant'} or any(len(v) != 1 for v in query.values()):
invalid()
if row['state'] != 'ready':
raise APIError(409, 'conflict', 'Вложение ещё не готово')
variant = query.get('variant', ['original'])[0]
if variant != 'original':
item = next((v for v in row['variants'] if v['name'] == variant), None)
if item is None:
missing()
row = dict(row, object_key=item['object_key'], content_type=item['content_type'], size_bytes=item['size_bytes'])
response = store.objects.get(row['object_key'])
try:
chunk = response['Body'].read(64 * 1024)
if response['ContentLength'] != row['size_bytes'] or not chunk:
unavailable()
except Exception:
response['Body'].close()
unavailable()
return 200, (row, response, chunk)
if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']:
chat_id = identifier(parts[4])
if store.access_mode == 'baseline':
store.chat(actor, chat_id)
if len(parts) == 5 and method == 'GET':
return 200, store.chat(actor, chat_id)
if len(parts) == 6 and parts[5] == 'attachments' and method == 'POST':
store.chat(actor, chat_id)
if set(query) != {'filename'} or len(query['filename']) != 1:
invalid()
name = filename(query['filename'][0])
with bounded_upload():
with receive(self) as (source, length, mime, digest):
return 202, store.upload(actor, chat_id, name, source, length, mime, digest)
if len(parts) == 6 and parts[5] == 'messages':
if method == 'POST':
keys = self.headers.get_all('Idempotency-Key', [])
return 201, store.send(actor, chat_id, self.body(), keys[0] if len(keys) == 1 else None)
if method == 'GET':
scope = 'messages:' + chat_id
limit, cursor = page_args(query, scope)
return 200, store.history(actor, chat_id, limit, cursor, scope)
missing()
do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch
class Server(http.server.HTTPServer):
request_queue_size = 64
allow_reuse_address = True
def __init__(self, address, store):
self.store = store
self.instance_id = tele.INSTANCE
workers = int(os.environ.get('HTTP_WORKERS', '16'))
if not 1 <= workers <= 128:
raise ValueError('HTTP_WORKERS must be 1..128')
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers)
self.slots = threading.BoundedSemaphore(workers * 2)
self.connections = set()
self.connections_lock = threading.Lock()
super().__init__(address, Handler)
def process_request(self, request, address):
if not self.slots.acquire(blocking=False):
rid = uuid.uuid4().hex
raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode()
try:
request.settimeout(0.2)
request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Instance-Id: ' + self.instance_id + '\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw)
except OSError:
pass
self.shutdown_request(request)
return
with self.connections_lock:
self.connections.add(request)
self.executor.submit(self.worker, request, address)
def worker(self, request, address):
try:
self.finish_request(request, address)
except OSError:
pass
finally:
self.shutdown_request(request)
with self.connections_lock:
self.connections.discard(request)
self.slots.release()
def close_connections(self):
with self.connections_lock:
connections = list(self.connections)
for connection in connections:
try:
connection.shutdown(socket.SHUT_RDWR)
except OSError:
pass
def server_close(self):
super().server_close()
# Bound draining even when a slow client keeps an incomplete request alive.
deadline = threading.Timer(5, self.close_connections)
deadline.start()
try:
self.executor.shutdown(wait=True)
finally:
deadline.cancel()
def main():
logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s')
tele.configure('messenger-api')
from postgres import Store
store = Store()
server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store)
stopping = threading.Event()
def stop(signum, frame):
if not stopping.is_set():
stopping.set()
LOG.info('shutdown_started signal=%s', signum)
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGUSR1, lambda *_: LOG.warning('pool_stats instance=%s stats=%s', server.instance_id, json.dumps(store.pool.get_stats())))
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
LOG.info('started port=%s', server.server_address[1])
try:
server.serve_forever(poll_interval=0.1)
finally:
server.server_close()
store.close()
tele.shutdown()
LOG.info('shutdown_complete')
if __name__ == '__main__':
main()
+217
View File
@@ -0,0 +1,217 @@
"""Allowlisted manual SDK telemetry; bounded exporters never carry business payloads."""
import contextlib
import contextvars
import datetime as dt
import json
import logging
import os
from pathlib import Path
import re
import resource
import threading
import time
import uuid
from opentelemetry import trace, metrics
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider, SpanProcessor
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
INSTANCE = uuid.uuid4().hex
ACTIVE = False
SUPPRESSED = contextvars.ContextVar('telemetry_suppressed', default=False)
PROVIDERS = []
COUNTERS = {}
HISTOGRAMS = {}
STATE = {'worker_busy': 0, 'queue_depth': 0, 'queue_oldest_age': 0, 'queue_observation_ok': 1,
'spans_finished': 0, 'spans_exported': 0, 'spans_export_failed': 0}
LOCK = threading.Lock()
PROPAGATOR = TraceContextTextMapPropagator()
LOGGER = logging.getLogger('messenger.telemetry')
TRACER = trace.get_tracer('messenger')
POOL = None
SERVICE = ''
BUCKETS = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 60]
class FinishedSpans(SpanProcessor):
def on_end(self, span):
with LOCK:
STATE['spans_finished'] += 1
class CountedExporter(SpanExporter):
def __init__(self, delegate):
self.delegate = delegate
def export(self, spans):
result = self.delegate.export(spans)
with LOCK:
STATE['spans_exported' if result == SpanExportResult.SUCCESS else 'spans_export_failed'] += len(spans)
return result
def shutdown(self):
self.delegate.shutdown()
def force_flush(self, timeout_millis=30000):
return self.delegate.force_flush(timeout_millis)
def configure(service):
global ACTIVE, TRACER, SERVICE
if PROVIDERS or os.environ.get('TELEMETRY_ENABLED', '1') == '0':
return
SERVICE = service
ratio = float(os.environ.get('TRACE_SAMPLE_RATIO', '0.25'))
if not 0 <= ratio <= 1:
raise ValueError('TRACE_SAMPLE_RATIO must be 0..1')
resource_attributes = Resource.create({
'service.name': service, 'service.version': 'lab6.1',
'service.instance.id': INSTANCE, 'deployment.environment.name': 'lab',
})
endpoint = os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
provider = TracerProvider(resource=resource_attributes, sampler=ParentBased(TraceIdRatioBased(ratio)))
provider.add_span_processor(FinishedSpans())
provider.add_span_processor(BatchSpanProcessor(
CountedExporter(OTLPSpanExporter(endpoint=endpoint+'/v1/traces', timeout=1)),
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=500,
export_timeout_millis=1500))
trace.set_tracer_provider(provider)
TRACER = trace.get_tracer('messenger')
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint+'/v1/metrics', timeout=1),
export_interval_millis=2000, export_timeout_millis=1500)
meter_provider = MeterProvider(resource=resource_attributes, metric_readers=[reader],
views=[View(instrument_name='*duration', aggregation=ExplicitBucketHistogramAggregation(BUCKETS))])
metrics.set_meter_provider(meter_provider)
meter = metrics.get_meter('messenger')
for name in ['http_server_requests', 'jobs_processed', 'job_retries']:
COUNTERS[name] = meter.create_counter(name)
for name in ['http_server_duration', 'job_processing_duration', 'job_ready_duration']:
HISTOGRAMS[name] = meter.create_histogram(name, unit='s')
def gauge(name, getter, unit=''):
def observe(options):
try:
yield metrics.Observation(getter(), {})
except (OSError, ValueError):
return
meter.create_observable_gauge(name, callbacks=[observe], unit=unit)
for name in ['queue_depth', 'queue_oldest_age', 'queue_observation_ok', 'worker_busy',
'spans_finished', 'spans_exported', 'spans_export_failed']:
gauge(name, lambda name=name: STATE[name], 's' if name=='queue_oldest_age' else '')
gauge('worker_capacity', lambda: int(os.environ.get('WORKER_CONCURRENCY', '1')) if service=='messenger-worker' else 0)
gauge('db_pool_used', lambda: max(0, POOL.get_stats().get('pool_size',0)-POOL.get_stats().get('pool_available',0)) if POOL else 0)
gauge('db_pool_waiting', lambda: POOL.get_stats().get('requests_waiting',0) if POOL else 0)
gauge('db_pool_capacity', lambda: int(os.environ.get('DB_POOL_MAX','8')) if POOL else 0)
gauge('process_resident_memory', lambda: int(Path('/proc/self/statm').read_text().split()[1])*os.sysconf('SC_PAGE_SIZE'), 'By')
gauge('container_memory', lambda: int(Path('/sys/fs/cgroup/memory.current').read_text()), 'By')
def cpu(options):
use = resource.getrusage(resource.RUSAGE_SELF)
yield metrics.Observation(use.ru_utime+use.ru_stime, {})
meter.create_observable_counter('process_cpu', callbacks=[cpu], unit='s')
log_provider = LoggerProvider(resource=resource_attributes)
log_provider.add_log_record_processor(BatchLogRecordProcessor(
OTLPLogExporter(endpoint=endpoint+'/v1/logs', timeout=1),
max_queue_size=256, max_export_batch_size=64, schedule_delay_millis=500,
export_timeout_millis=1500))
LOGGER.setLevel(logging.INFO)
LOGGER.propagate = False
LOGGER.addHandler(LoggingHandler(level=logging.INFO, logger_provider=log_provider))
PROVIDERS.extend([provider, meter_provider, log_provider])
ACTIVE = True
def pool(value):
global POOL
POOL = value
def route(path):
if path in ['/health/live', '/health/ready', '/api/v1/users', '/api/v1/users/me',
'/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats']:
return path
patterns = [
(r'/api/v1/chats/[^/]+/messages', '/api/v1/chats/{chat_id}/messages'),
(r'/api/v1/chats/[^/]+/attachments', '/api/v1/chats/{chat_id}/attachments'),
(r'/api/v1/chats/[^/]+', '/api/v1/chats/{chat_id}'),
(r'/api/v1/attachments/[^/]+/content', '/api/v1/attachments/{attachment_id}/content'),
(r'/api/v1/attachments/[^/]+', '/api/v1/attachments/{attachment_id}'),
]
return next((template for pattern,template in patterns if re.fullmatch(pattern,path)), 'unmatched')
@contextlib.contextmanager
def span(name, attributes=None, kind=SpanKind.INTERNAL, carrier=None):
if not ACTIVE or SUPPRESSED.get() or (name == 'db.transaction' and not trace.get_current_span().get_span_context().is_valid):
yield trace.get_current_span()
return
context = PROPAGATOR.extract(carrier) if carrier is not None else None
with TRACER.start_as_current_span(name, attributes=attributes or {}, kind=kind,
context=context, record_exception=False, set_status_on_exception=False) as current:
try:
yield current
except Exception as error:
current.set_attribute('error.type', type(error).__name__)
current.set_status(Status(StatusCode.ERROR))
log('operation_failed', error_type=type(error).__name__)
raise
@contextlib.contextmanager
def suppressed():
token = SUPPRESSED.set(True)
try:
yield None
finally:
SUPPRESSED.reset(token)
def carrier():
result = {}
if ACTIVE:
PROPAGATOR.inject(result)
return result
def log(event, **attributes):
if not ACTIVE:
return
# Callers pass only fixed event names, bounded outcomes and technical correlation IDs.
context = trace.get_current_span().get_span_context()
body = {'timestamp': dt.datetime.now(dt.timezone.utc).isoformat(),
'severity': 'INFO', 'service.name': SERVICE, 'event': event,
'sampled': bool(context.trace_flags.sampled), 'trace_id': format(context.trace_id,'032x'), 'span_id': format(context.span_id,'016x'),
**attributes}
LOGGER.info(json.dumps(body, ensure_ascii=True, separators=(',',':')))
def count(name, attributes=None):
if ACTIVE:
COUNTERS[name].add(1, attributes or {})
def duration(name, seconds, attributes=None):
if ACTIVE:
HISTOGRAMS[name].record(max(0, seconds), attributes or {})
def flush():
for provider in PROVIDERS:
provider.force_flush(timeout_millis=5000)
def shutdown():
for provider in PROVIDERS:
provider.shutdown()
class S3Client:
"""Span each SDK request without serializing args, keys, URLs or SDK exceptions."""
def __init__(self, client):
self.client = client
def __getattr__(self, name):
method = getattr(self.client,name)
if name not in {'head_object','get_object','put_object','delete_object','head_bucket','create_bucket','list_objects_v2'}:
return method
def call(*args, **kwargs):
with span('s3.'+name, {'rpc.system':'aws-api','rpc.service':'S3','rpc.method':name}, SpanKind.CLIENT):
return method(*args, **kwargs)
return call
+52
View File
@@ -0,0 +1,52 @@
"""Stop worker, accept queued, restart API/broker, then complete the same attachment."""
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
from client import Client,Failure
def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip()
def main():
c=Client();alice,bob,eve,chat=c.bootstrap();record={}
before=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True))
try:
docker('compose','stop','worker','dispatcher')
queued,raw=c.attachment(alice,chat)
assert queued['status']=='queued'
message=c.message(alice,chat,'accepted with worker stopped',attachments=[queued['id']])['json']
c.call('GET','/api/v1/attachments/'+queued['id']+'/content',409,actor=bob)
killed=docker('compose','ps','-q','api').splitlines()
for container in killed:docker('kill','--signal=KILL',container)
record['killed_api_containers']=killed
docker('compose','up','-d','--wait','--no-deps','api')
time.sleep(2)
assert c.call('GET','/api/v1/attachments/'+queued['id'],actor=bob,schema='Attachment')['json']['status']=='queued'
docker('compose','up','-d','--no-deps','dispatcher')
deadline=time.monotonic()+10
while int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))<1:
assert time.monotonic()<deadline;time.sleep(.1)
docker('compose','stop','dispatcher')
length_before=int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))
docker('compose','restart','broker')
docker('compose','up','-d','--wait','--no-deps','broker')
length_after=int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))
assert length_after>=length_before>=1
record['stream_entries_before_after_broker_restart']=[length_before,length_after]
docker('compose','up','-d','--no-deps','worker','dispatcher')
ready=c.wait_ready(bob,queued)
assert c.call('GET','/api/v1/attachments/'+queued['id']+'/content',actor=bob)['bytes']==raw
record.update(attachment_id=queued['id'],original_sha256=hashlib.sha256(raw).hexdigest(),status=ready['status'],variants=[v['name'] for v in ready['variants']],old_bearer_survived=True,message_attachment_retained=message['attachment_ids']==[queued['id']])
finally:docker('compose','up','-d','--no-deps','worker','dispatcher')
after=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True))
record['before']=before;record['after']=after
print(json.dumps(record,indent=2))
if __name__=='__main__':main()
+24
View File
@@ -0,0 +1,24 @@
"""Small deterministic PNG helpers; generation happens outside measurement."""
import random
import struct
import zlib
def chunk(name, data):
return struct.pack('!I', len(data)) + name + data + struct.pack('!I', zlib.crc32(name + data) & 0xffffffff)
def png(width, height, noise=False, rgb=True):
channels = 3 if rgb else 1
rand = random.Random(2301)
encoded = zlib.compressobj(1)
parts = []
for _ in range(height):
line = bytes(rand.getrandbits(8) for _ in range(width * channels)) if noise else b'\x7f' * (width * channels)
parts.append(encoded.compress(b'\0' + line))
parts.append(encoded.flush())
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2 if rgb else 0, 0, 0, 0)) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'')
def bomb():
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', 10000, 4001, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'x')) + chunk(b'IEND', b'')
+257
View File
@@ -0,0 +1,257 @@
"""Real OTLP evidence, controlled incidents and bounded telemetry-backend outage."""
import argparse
import base64
import concurrent.futures
import json
import os
from pathlib import Path
import subprocess
import sys
import time
from urllib.parse import urlencode
import urllib.request
import uuid
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, Failure
BASE=os.environ.get('BASE_URL','http://127.0.0.1:18093')
PROM=os.environ.get('PROMETHEUS_URL','http://127.0.0.1:19096')
LOKI=os.environ.get('LOKI_URL','http://127.0.0.1:13106')
TEMPO=os.environ.get('TEMPO_URL','http://127.0.0.1:13206')
COLLECTOR=os.environ.get('COLLECTOR_URL','http://127.0.0.1:18896')
OPENER=urllib.request.build_opener(urllib.request.ProxyHandler({}))
def get(url):
req=urllib.request.Request(url,headers={'Accept':'application/json'})
with OPENER.open(req,timeout=10) as response:
return json.load(response)
def prom(query):
return get(PROM+'/api/v1/query?'+urlencode({'query':query}))['data']['result']
def scalar(query):
rows=prom(query)
return float(rows[0]['value'][1]) if rows else 0
def logs(query, start=None, limit=2000):
params={'query':query,'start':str(int((start or time.time()-600)*1e9)),
'end':str(time.time_ns()),'limit':str(limit),'direction':'forward'}
result=get(LOKI+'/loki/api/v1/query_range?'+urlencode(params))['data']['result']
return [json.loads(value) for stream in result for stamp,value in stream['values']]
def trace(identifier):
return get(TEMPO+'/api/traces/'+identifier)
def spans(data):
result=[]
for batch in data.get('batches',data.get('resourceSpans',[])):
service=next(a['value']['stringValue'] for a in batch['resource']['attributes'] if a['key']=='service.name')
for scope in batch.get('scopeSpans',[]):
for span in scope['spans']:
result.append(dict(span,service=service))
return result
def wait(function,timeout=60):
end=time.monotonic()+timeout
while time.monotonic()<end:
try:
result=function()
if result:return result
except (OSError,ValueError,Failure):
pass
time.sleep(.5)
raise AssertionError('Timed out: '+function.__name__)
def compose(*args):
return subprocess.check_output(['docker','compose',*args],text=True).strip()
def save(name,data):
(ROOT/'evidence'/name).write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n')
def upload(c,actor,chat,tid):
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=private-canary.png',202,
actor=actor,raw=raw,headers={'Content-Type':'image/png',
'traceparent':'00-'+tid+'-0123456789abcdef-01'},schema='Attachment')
def proof():
wait(lambda:get(LOKI+'/loki/api/v1/labels'))
wait(lambda:get(TEMPO+'/api/search'))
proof_start=time.time()
c=Client(BASE);a,b,e,chat=c.bootstrap()
tid=uuid.uuid4().hex
marker='SECRET_MESSAGE_CANARY_'+uuid.uuid4().hex
c.message(a,chat,marker)
response=upload(c,a,chat,tid)
ready=c.wait_ready(b,response['json'])
def completed_trace():
data=trace(tid)
return data if any(s['name']=='attachment process' for s in spans(data)) else None
data=wait(completed_trace)
rows=spans(data)
upload_span=next(s for s in rows if s['kind']=='SPAN_KIND_SERVER')
publish=next(s for s in rows if s['kind']=='SPAN_KIND_PRODUCER')
consumer=next(s for s in rows if s['kind']=='SPAN_KIND_CONSUMER')
assert consumer['parentSpanId']==publish['spanId']
index={s['spanId']:s for s in rows}
parent=publish
while parent['spanId']!=upload_span['spanId']:
parent=index[parent['parentSpanId']]
assert any(s['service']=='messenger-worker' and s['name'].startswith('s3.') for s in rows)
assert any(s['service']=='messenger-worker' and s['name']=='db.transaction' for s in rows)
query='{service_name=~"messenger-.*"} | json | trace_id="'+tid+'"'
records=wait(lambda: logs(query) if len(logs(query))>=4 else None)
assert {r['event'] for r in records}>={'http_request','attachment_accepted','job_published','job_processed'}
assert any(r.get('request_id')==response['headers'].get('X-Request-Id') for r in records)
assert all(len(r['trace_id'])==32 and len(r['span_id'])==16 for r in records)
all_logs=logs('{service_name=~"messenger-.*"}',start=proof_start,limit=5000)
payload=json.dumps(all_logs)+json.dumps(data)
for value in [marker,a['token'],a['registration']['password'],a['registration']['username'],'private-canary.png']:
assert value not in payload, 'business/credential canary leaked'
metric_rows=prom('{service_name=~"messenger-.*"}')
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','job_id','trace_id','span_id'}
assert all(not forbidden.intersection(row['metric']) for row in metric_rows)
names={r['metric']['__name__'] for r in metric_rows}
assert {'http_server_duration_seconds_bucket','jobs_processed_total','queue_depth','queue_oldest_age_seconds','db_pool_used','worker_capacity'}<=names
save('trace-proof.json',{'trace_id':tid,'request_id':response['headers'].get('X-Request-Id'),
'status':ready['status'],'consumer_parent_matches_producer':True,'producer_descends_from_upload':True,
'api_worker_logs_correlated':True,'secret_canaries_absent':True,'metric_labels_allowlisted':True,
'trace':data,'logs':records})
# Retry is a real S3 outage after durable acceptance, not fabricated spans.
compose('stop','worker')
retry_tid=uuid.uuid4().hex
queued=upload(c,a,chat,retry_tid)['json']
paused=False
try:
compose('pause','s3');paused=True
compose('up','-d','--no-deps','worker')
def attempted():
raw=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',
"SELECT attempt FROM attachment_jobs WHERE 1=1 AND attachment_id = '"+queued['id']+"';")
return int(raw)>=1
wait(attempted)
time.sleep(5)
finally:
if paused:compose('unpause','s3')
compose('up','-d','--no-deps','worker')
wait(lambda:c.call('GET','/health/ready',schema='Health'))
c.wait_ready(a,queued)
def retried():
data=trace(retry_tid)
return data if len([s for s in spans(data) if s['kind']=='SPAN_KIND_CONSUMER'])>=2 else None
retry_data=wait(retried)
retry_records=wait(lambda:logs('{service_name="messenger-worker"} | json | trace_id="'+retry_tid+'" | event="job_retry"'))
save('retry-trace.json',{'trace_id':retry_tid,'retry_observed':True,'trace':retry_data,'logs':retry_records})
print('PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.')
def alerts():
return get(PROM+'/api/v1/alerts')['data']['alerts']
def firing(name):
return [a for a in alerts() if a['labels']['alertname']==name and a['state']=='firing']
def incidents():
c=Client(BASE);a,b,e,chat=c.bootstrap();result={}
start=time.time()
compose('stop','worker')
try:
queued=[c.attachment(a,chat)[0] for _ in range(3)]
observed=wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')>=3)
fired=wait(lambda:firing('QueueLag'))
detected=time.time()
metrics={q:prom(q) for q in ['queue_depth','queue_oldest_age_seconds','worker_busy','worker_capacity']}
result['worker_stop']={'injected_at':start,'detected_at':detected,
'detection_seconds':detected-start,'firing':fired,'metrics':metrics,
'diagnosis':'Backlog and oldest age grow; worker completion stops while API acceptance still succeeds. Gauges expire after 30s.'}
finally:
compose('up','-d','--no-deps','worker')
for att in queued:c.wait_ready(a,att)
wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')==0)
wait(lambda:not firing('QueueLag'))
result['worker_stop']['resolved_at']=time.time()
# Keep readiness SELECT 1 available while real session SQL is blocked.
lock=subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1',
'-c',"BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(35); COMMIT;"],
stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
time.sleep(1)
db_start=time.time();stop=False;errors=[];error_trace=uuid.uuid4().hex
def traffic():
while not stop:
r=c.call('GET','/api/v1/users/me',[200,503],actor=a,
headers={'traceparent':'00-'+error_trace+'-0123456789abcdef-01'})
if r['status']==503:errors.append(r['elapsed'])
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
futures=[pool.submit(traffic) for _ in range(4)]
try:
fired=wait(lambda:firing('APIHighErrorRate'),timeout=50)
detected=time.time()
result['db_lock']={'injected_at':db_start,'detected_at':detected,'detection_seconds':detected-db_start,
'firing':fired,'error_latencies_seconds':errors.copy(),'trace_id':error_trace}
lock.wait(timeout=45)
finally:
stop=True
for future in futures:future.result()
finally:
stop=True
if lock.poll() is None:
lock.terminate();lock.wait(timeout=5)
wait(lambda:c.call('GET','/api/v1/users/me',actor=a,schema='User'))
wait(lambda:not firing('APIHighErrorRate'),timeout=70)
data=wait(lambda:trace(error_trace))
assert any(s['name']=='db.transaction' and s.get('status',{}).get('code')=='STATUS_CODE_ERROR' for s in spans(data))
result['db_lock'].update(resolved_at=time.time(),trace=data,
diagnosis='Server 5xx spans contain slow/error PostgreSQL transaction; health remains available; sessions lock is the bottleneck.')
save('incidents.json',result)
print('PASS: worker-stop and real DB-lock incidents, local firing/resolved and causal DB traces.')
def collector_metrics():
with OPENER.open(COLLECTOR+'/metrics',timeout=5) as r:raw=r.read().decode()
result={}
for line in raw.splitlines():
if line.startswith('otelcol_') and not line.startswith('#'):
name=line.split('{')[0].split(' ')[0]
try:result[name]=result.get(name,0)+float(line.rsplit(' ',1)[-1])
except ValueError:pass
return result
def backend_outage():
c=Client(BASE);a,b,e,chat=c.bootstrap();before=collector_metrics()
cid=compose('ps','-q','otel-collector')
samples=[];messages=[];attachments=[];start=time.time()
compose('stop','tempo')
try:
for i in range(45):
for _ in range(3):
messages.append(c.message(a,chat,'synthetic-backend-outage')['json']['id'])
if i%10==0:
att,_=c.attachment(a,chat);attachments.append(c.wait_ready(a,att)['id'])
stat=json.loads(subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',cid],text=True))
samples.append({'elapsed':time.time()-start,'memory':stat['MemUsage'],'cpu':stat['CPUPerc']})
time.sleep(1)
during=collector_metrics()
assert during.get('otelcol_exporter_send_failed_spans_total',0)>before.get('otelcol_exporter_send_failed_spans_total',0)
assert during.get('otelcol_exporter_queue_size',0)<=128*2
history=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100',actor=b,schema='MessagePage')['json']
got={m['id'] for m in history['items']}
if history['next_cursor']:
history2=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+history['next_cursor'],actor=b,schema='MessagePage')['json']
got.update(m['id'] for m in history2['items'])
assert set(messages)<=got
finally:compose('up','-d','--no-deps','tempo')
wait(lambda:get(TEMPO+'/api/search'))
tid=uuid.uuid4().hex
upload(c,a,chat,tid)
wait(lambda:trace(tid))
save('backend-outage.json',{'duration_seconds':time.time()-start,'confirmed_messages':len(messages),
'confirmed_attachments_ready':len(attachments),'all_confirmed_messages_retained':True,
'collector_samples':samples,'before':before,'during':during,'recovered_trace_id':tid})
print('PASS: Tempo unavailable, finite Collector buffers/drops, API writes/jobs retained, fresh trace after recovery.')
if __name__=='__main__':
parser=argparse.ArgumentParser()
parser.add_argument('phase',choices=['proof','incidents','backend'])
args=parser.parse_args()
{'proof':proof,'incidents':incidents,'backend':backend_outage}[args.phase]()
+193
View File
@@ -0,0 +1,193 @@
"""Fixed-arrival mixed workload: SDK off, full traces, 10% and 25% head sampling."""
import concurrent.futures
import http.client
import json
import os
from pathlib import Path
import socket
import statistics
import subprocess
import sys
import threading
import time
from urllib.parse import urlsplit
import uuid
sys.path.insert(0,str(Path(__file__).resolve().parent))
from observability import Client, BASE, compose, collector_metrics, scalar, prom, logs, wait, save
class DockerHTTP(http.client.HTTPConnection):
def connect(self):
self.sock=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM)
self.sock.settimeout(8)
self.sock.connect(DOCKER_SOCKET)
context=json.loads(subprocess.check_output(['docker','context','inspect'],text=True))[0]
DOCKER_SOCKET=urlsplit(context['Endpoints']['docker']['Host']).path
def stats(container):
connection=DockerHTTP('localhost')
try:
connection.request('GET','/containers/'+container+'/stats?stream=false&one-shot=true')
response=connection.getresponse()
assert response.status==200
data=json.load(response)
finally:connection.close()
memory=data.get('memory_stats',{})
details=memory.get('stats',{})
usage=memory.get('usage',0)
return {'cpu_seconds':data['cpu_stats']['cpu_usage']['total_usage']/1e9,
'anonymous_bytes':details.get('anon',details.get('rss',0)),
'working_set_bytes':max(0,usage-details.get('inactive_file',details.get('total_inactive_file',0))),
'rx_bytes':sum(n.get('rx_bytes',0) for n in data.get('networks',{}).values())}
def span_counts():
return {r['metric']['service_instance_id']:float(r['value'][1])
for r in prom('spans_finished{service_name=~"messenger-.*"}')}
def delta(previous,current):
return sum(max(0,value-previous.get(key,0)) for key,value in current.items())
def quantile(values,q):
return sorted(values)[min(len(values)-1,int((len(values)-1)*q))]
def set_mode(enabled,ratio):
env=os.environ.copy()
env.update(TELEMETRY_ENABLED=str(enabled),TRACE_SAMPLE_RATIO=str(ratio))
subprocess.run(['docker','compose','up','-d','--wait','--no-deps','api','worker','dispatcher'],env=env,
check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
c=Client(BASE)
wait(lambda:c.call('GET','/health/ready',schema='Health'))
return c
def probes(c,actor,mode):
results=[]
for duration in [3.5,3.5,3.5,1.2]:
query="BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep("+str(duration)+"); COMMIT;"
lock=subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',query],
stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
try:
def locked():
value=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',
"SELECT count(*) FROM pg_locks WHERE 1=1 AND relation = 'sessions'::regclass AND mode = 'AccessExclusiveLock' AND granted;")
return int(value)>0
wait(locked,timeout=5)
result=c.call('GET','/api/v1/users/me',[200,503],actor=actor)
assert result['status']==(503 if duration>3 else 200)
assert result['elapsed']>.1
results.append({'request_id':result['headers'].get('X-Request-Id'),
'status':result['status'],'latency_seconds':result['elapsed']})
finally:lock.wait(timeout=5)
if mode!='off':
time.sleep(3)
for row in results:
records=logs('{service_name="messenger-api"} | json | request_id="'+row['request_id']+'"',limit=10)
assert records
row['trace_sampled']=records[0]['sampled']
row['trace_id']=records[0]['trace_id']
else:
for row in results:row['trace_sampled']=False
return results
def run(mode,enabled,ratio,repeat,seconds,rps):
c=set_mode(enabled,ratio);a,b,e,chat=c.bootstrap()
for _ in range(10):c.call('GET','/api/v1/users/me',actor=a)
time.sleep(5)
names=compose('ps','-q','api','worker','dispatcher','otel-collector','postgres','loki','tempo','prometheus','grafana','s3','s3-store','broker','lb','redis').splitlines()
labels={}
for cid in names:
label=json.loads(subprocess.check_output(['docker','inspect',cid,'--format','{{json .Config.Labels}}'],text=True))
labels[cid]=label['com.docker.compose.service']
before={cid:stats(cid) for cid in names}
ingest_before=collector_metrics();spans_before=span_counts()
memory={cid:[] for cid in names};sampling_stop=threading.Event()
def sample():
while not sampling_stop.is_set():
for cid in names:
try:memory[cid].append(stats(cid))
except OSError:pass
sampling_stop.wait(1)
sampler=threading.Thread(target=sample,daemon=True);sampler.start()
latencies=[];failures=[];attachments=[];start=time.monotonic()
def request(i):
try:
if i%20==0:
result=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=cost.png',202,
actor=a,raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes(),
headers={'Content-Type':'image/png'},schema='Attachment')
return result['elapsed'],result['json']
if i%5==0:
result=c.message(a,chat,'fixed-synthetic-cost-payload')
else:result=c.call('GET','/api/v1/users/me',actor=a,schema='User')
return result['elapsed'],None
except Exception as error:return None,type(error).__name__
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool:
futures=[]
for i in range(seconds*rps):
delay=start+i/rps-time.monotonic()
if delay>0:time.sleep(delay)
futures.append(pool.submit(request,i))
for future in futures:
elapsed,att=future.result()
if elapsed is None:failures.append(att)
else:
latencies.append(elapsed)
if att:attachments.append(att)
load_wall=time.monotonic()-start
for att in attachments:c.wait_ready(a,att)
# Drain exporter batches and metric-reader intervals, then compare counters by instance.
time.sleep(5)
sampling_stop.set();sampler.join(timeout=15)
after={cid:stats(cid) for cid in names}
ingest_after=collector_metrics();spans_after=span_counts()
assert not failures, failures
resource={}
for cid in names:
service=labels[cid]
target=resource.setdefault(service,{'cpu_seconds':0,'peak_sampled_anonymous_bytes':0,'peak_sampled_working_set_bytes':0,'rx_bytes':0})
target['cpu_seconds']+=max(0,after[cid]['cpu_seconds']-before[cid]['cpu_seconds'])
target['rx_bytes']+=max(0,after[cid]['rx_bytes']-before[cid]['rx_bytes'])
target['peak_sampled_anonymous_bytes']+=max((s['anonymous_bytes'] for s in memory[cid]),default=0)
target['peak_sampled_working_set_bytes']+=max((s['working_set_bytes'] for s in memory[cid]),default=0)
counts={k:max(0,ingest_after.get(k,0)-ingest_before.get(k,0)) for k in [
'otelcol_receiver_accepted_spans_total','otelcol_receiver_accepted_log_records_total',
'otelcol_receiver_accepted_metric_points_total','otelcol_exporter_sent_spans_total',
'otelcol_exporter_send_failed_spans_total']}
finished=delta(spans_before,spans_after)
accepted=counts['otelcol_receiver_accepted_spans_total']
result={'mode':mode,'enabled':enabled,'sample_ratio':ratio,'repeat':repeat,
'arrival_rps':rps,'scheduled_requests':seconds*rps,'confirmed_requests':len(latencies),
'load_wall_seconds':load_wall,'successful_rps':len(latencies)/load_wall,
'p50_seconds':quantile(latencies,.5),'p95_seconds':quantile(latencies,.95),'p99_seconds':quantile(latencies,.99),
'errors':failures,'image_jobs_ready':len(attachments),'resource':resource,'collector_ingest':counts,
'sampled_spans_finished':finished,'sampled_spans_not_accepted_after_drain':max(0,finished-accepted)}
result['rare_probes']=probes(c,a,mode)
print(mode,repeat,'p95',round(result['p95_seconds'],5),'ingested spans',accepted,flush=True)
return result
def main():
import argparse
parser=argparse.ArgumentParser()
parser.add_argument('--seconds',type=int,default=20)
parser.add_argument('--rps',type=int,default=30)
parser.add_argument('--repeats',type=int,default=3)
args=parser.parse_args()
result={'method':'Fixed arrival mixed requests, 5% image uploads, 15% messages, 80% profile reads; same fixture/resources. No transparent retries.',
'notes':['CPU cgroup total across load plus exporter drain; includes renderer.',
'Anonymous/working-set memory sampled at 1Hz through read-only Docker stats API, not exact per-PID peak RSS.',
'Collector rx bytes include OTLP HTTP, Prometheus scrapes and backend replies; accepted signal counts are exact Collector counter deltas.',
'Fresh API/worker/dispatcher containers per setting; data/backends retained. Per-instance counter deltas ignore expired old series.',
'Rare SQL-lock probes run after resource/ingest measurement; three SQL-lock probes and one slow success per run; future runs synchronize on pg_locks.'],
'runs':[]}
modes=[('off',0,0),('full',1,1),('sample10',1,.1),('sample25',1,.25)]
try:
for repeat in range(args.repeats):
# Rotate order to reduce a fixed warm-cache/time-order advantage.
order=modes[repeat%len(modes):]+modes[:repeat%len(modes)]
for mode,enabled,ratio in order:
result['runs'].append(run(mode,enabled,ratio,repeat,args.seconds,args.rps))
save('observability-cost.json',result)
finally:set_mode(1,.25)
print('PASS: telemetry cost experiment completed; balanced 25% tracing restored.')
if __name__=='__main__':main()
+37
View File
@@ -0,0 +1,37 @@
"""Restart all stores and verify historical telemetry alongside public business persistence."""
import json
import os
from pathlib import Path
import subprocess
import time
from urllib.parse import urlencode
from observability import BASE,PROM,trace,spans,logs,get,wait,save
root=Path(__file__).resolve().parents[1]
identifier=json.loads((root/'evidence/trace-proof.json').read_text())['trace_id']
before_trace=trace(identifier)
before_ids={s['spanId'] for s in spans(before_trace)}
query='{service_name=~"messenger-.*"} | json | trace_id="'+identifier+'"'
before_logs=logs(query,start=time.time()-3600)
assert before_logs
at=time.time()-5
url=PROM+'/api/v1/query?'+urlencode({'query':'sum(http_server_requests_total)','time':str(at)})
before_metric=get(url)['data']['result']
assert before_metric
command=['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh','BASE_URL='+BASE]
result=subprocess.run(command,capture_output=True,text=True)
(root/'evidence/persistence.txt').write_text(result.stdout+result.stderr)
assert result.returncode==0,result.stdout[-300:]
after_trace=wait(lambda:trace(identifier),timeout=90)
after_ids={s['spanId'] for s in spans(after_trace)}
assert before_ids<=after_ids
after_logs=wait(lambda:logs(query,start=time.time()-3600),timeout=90)
assert {json.dumps(r,sort_keys=True) for r in before_logs}<={json.dumps(r,sort_keys=True) for r in after_logs}
after_metric=wait(lambda:get(url)['data']['result'])
assert before_metric==after_metric
save('telemetry-persistence.json',{'trace_id':identifier,'historical_span_count':len(before_ids),
'historical_logs':len(before_logs),'fixed_metric_query_time':at,
'metric_before':before_metric,'metric_after':after_metric,
'business_data_and_existing_session_survived':True,'all_telemetry_volumes_restarted':True,
'trace_log_metric_history_survived':True})
print('PASS: full business restart, original Bearer, historical Tempo trace, Loki logs and fixed-time Prometheus sample.')
+188
View File
@@ -0,0 +1,188 @@
"""Real DB/S3/Redis fault injection. Stop production worker/dispatcher first."""
import concurrent.futures
import hashlib
import io
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
sys.path[:0]=['/app/src',str(Path.cwd()/'tests')]
from PIL import Image
from client import Client
from pipeline import STREAM,GROUP,broker,connect,event,dispatch_once,process,ack
from objects import Objects
from postgres import setting
from admin import replay,version,cleanup
def main():
c=Client();alice,bob,eve,chat=c.bootstrap()
bus=broker(); objects=Objects(setting)
report={'invariants':{},'faults':[]}
def fetch(aid):
with connect() as conn:
a=conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s',(aid,)).fetchone()
j=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(a['current_job_id'],)).fetchone()
return a,j
def upload(raw=None,mime='image/png'):
if raw is None: a,_=c.attachment(alice,chat)
else:a=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=research.png',202,actor=alice,raw=raw,headers={'Content-Type':mime},schema='Attachment')['json']
return a,fetch(a['id'])[1]
def run_once(fault=None):
args=[sys.executable,'/app/src/pipeline.py','work','--once']
if fault:args+=['--fault-point',fault]
return subprocess.run(args,capture_output=True,timeout=30).returncode
def drain():
for _ in range(200):
batch=bus.xreadgroup(GROUP,'research-drain',{STREAM:'>'},count=1)
if not batch:break
for mid,data in batch[0][1]:process(json.loads(data['event']));ack(bus,mid)
def expire(job):
with connect() as conn:
conn.execute("UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE 1=1 AND id = %s",(job['id'],))
def recover(aid):
until=time.monotonic()+20
while time.monotonic()<until:
dispatch_once(bus);drain()
a,j=fetch(aid)
if a['state']=='ready':return a,j
time.sleep(.1)
raise AssertionError('not recovered')
for mid,data in bus.xautoclaim(STREAM,GROUP,'initial-recovery',min_idle_time=0,start_id='0-0',count=100)[1]:
process(json.loads(data['event']));ack(bus,mid)
# COMMIT -> no publisher yet: one queued row, no Redis message, then automatic outbox pass.
a,j=upload()
assert fetch(a['id'])[0]['state']=='queued'
assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',409,actor=bob)
c.message(bob,chat,'queued is linkable',attachments=[a['id']])
dispatch_once(bus);drain();assert fetch(a['id'])[0]['state']=='ready'
report['invariants']['commit_before_publish_recovered']=True
# Broker accepted XADD but publisher died before marking outbox: true exit(77).
a,j=upload()
code=subprocess.run([sys.executable,'/app/src/pipeline.py','dispatch','--once','--fault-point','after_publish'],capture_output=True,timeout=10).returncode
assert code==77
dispatch_once(bus);drain()
assert fetch(a['id'])[0]['state']=='ready'
report['invariants']['publish_before_marker_duplicate_safe']=True
for point in ['before_side_effect','after_original','after_objects','after_commit']:
a,j=upload();dispatch_once(bus)
code=run_once(point);assert code==77,(point,code)
before,job=fetch(a['id'])
if point not in ('after_commit','before_side_effect'):expire(job)
result,job=recover(a['id'])
assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',actor=bob)['bytes']==(Path.cwd()/'tests/fixtures/sample.png').read_bytes()
reclaimed=bus.xautoclaim(STREAM,GROUP,'post-crash-redelivery',min_idle_time=0,start_id='0-0',count=100)[1]
for mid,data in reclaimed:
process(json.loads(data['event']));ack(bus,mid)
assert bus.xpending(STREAM,GROUP)['pending']==0
keys=objects.client.list_objects_v2(Bucket=objects.bucket,Prefix='results/'+a['id']+'/v1/').get('Contents',[])
assert len(keys)==4,(point,len(keys))
report['faults'].append({'point':point,'exit_code':code,'state_at_crash':before['state'],'recovered':result['state'],'attempts':job['attempt'],'result_objects':len(keys)})
report['invariants']['crashes_before_after_side_effect_and_commit']=True
bus.xadd(STREAM,{'event':json.dumps({'not':'a job'})})
assert run_once()==0
assert bus.xlen('rejected-jobs')>=1
report['invariants']['invalid_envelope_observable_dlq']=True
# Duplicate/reorder through broker on a completed job: no extra attempt/effect.
a,j=upload();dispatch_once(bus);drain();before,done=fetch(a['id'])
old=event(j)
for attempt in [2,1,3,1,1]:bus.xadd(STREAM,{'event':json.dumps(dict(old,attempt=attempt))})
drain();after,done2=fetch(a['id'])
assert before['variants']==after['variants'] and done2['attempt']==done['attempt']
report['invariants']['duplicate_and_reorder_no_second_effect']=True
# Race two consumers with the SAME logical job, not just one serial duplicate.
a,j=upload();e=event(j)
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: results=list(pool.map(lambda _:process(e),range(2)))
assert fetch(a['id'])[0]['state']=='ready' and sorted(results)==['noop','ready']
report['invariants']['concurrent_duplicate_single_claim']=True
# Pause actual old process after all immutable side effects, advance version and finish v2.
a,j=upload();e=event(j)
directory=Path('/tmp')/('fence-'+uuid.uuid4().hex);directory.mkdir();marker=directory/'paused';release=directory/'release'
code="import sys,json,time;from pathlib import Path;sys.path.insert(0,'/app/src');from pipeline import process\ndef hook(point,job):\n if point=='after_objects':\n Path(sys.argv[2]).write_text('paused')\n while not Path(sys.argv[3]).exists():time.sleep(.05)\nprint(process(json.loads(sys.argv[1]),hook))"
child=subprocess.Popen([sys.executable,'-c',code,json.dumps(e),str(marker),str(release)],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
try:
deadline=time.monotonic()+20
while not marker.exists():
assert child.poll() is None
assert time.monotonic()<deadline;time.sleep(.05)
new=version(uuid.UUID(a['id']),2);newrow,newjob=fetch(a['id'])
assert process(event(newjob))=='ready'
current,finished=fetch(a['id']);signature=current['variants']
release.write_text('go');stdout,stderr=child.communicate(timeout=10)
assert child.returncode==0 and stdout.strip()=='fenced',(stdout,stderr)
bus.xadd(STREAM,{'event':json.dumps(e)});drain()
final,finished=fetch(a['id'])
assert final['pipeline_version']==2 and final['variants']==signature and final['state']=='ready'
assert all('/v2/' in v['object_key'] for v in final['variants'])
report['invariants']['stale_running_version_fenced']=True
report['invariants']['old_delivery_after_new_ready_no_overwrite']=True
finally:
release.write_text('go')
if child.poll() is None:child.kill();child.wait()
# Transient failures consume a finite budget; failed registry and explicit replay.
a,j=upload()
with connect() as conn:conn.execute('UPDATE attachment_jobs SET max_attempts = 2 WHERE 1=1 AND id = %s',(j['id'],))
# Exact staging copy saved externally; missing source is permanent and observable.
raw=(Path.cwd()/'tests/fixtures/sample.png').read_bytes();objects.delete(j['source_key'])
process(event(j));failed,fj=fetch(a['id']);assert failed['state']=='failed' and failed['error_code']=='source_missing'
objects.put(j['source_key'],io.BytesIO(raw),len(raw),'image/png',hashlib.sha256(raw).hexdigest())
replay(j['id']);fresh,fj=fetch(a['id']);assert process(event(fj))=='ready'
try:replay(j['id']);raise AssertionError('replayed ready')
except ValueError:pass
report['invariants']['failed_registry_explicit_replay_and_ready_refusal']=True
# Real S3 signature failures are transient, with two attempts and measured backoff.
a,j=upload()
with connect() as conn:conn.execute('UPDATE attachment_jobs SET max_attempts = 2 WHERE 1=1 AND id = %s',(j['id'],))
wrong=Path('/tmp/wrong-secret');wrong.write_text('synthetic-invalid-key-not-a-real-secret')
env=dict(os.environ,S3_SECRET_ACCESS_KEY_FILE=str(wrong))
transitions=[];started=time.monotonic()
for _ in range(2):
row,j=fetch(a['id'])
delay=max(0,(j['next_attempt_at']-__import__('datetime').datetime.now(__import__('datetime').timezone.utc)).total_seconds())
time.sleep(delay+.01)
code="import sys,json;sys.path.insert(0,'/app/src');from pipeline import process;print(process(json.loads(sys.argv[1])))"
run=subprocess.run([sys.executable,'-c',code,json.dumps(event(j))],env=env,capture_output=True,timeout=10)
assert run.returncode==0
row,j=fetch(a['id']);transitions.append({'state':j['state'],'attempt':j['attempt'],'error_code':j['error_code']})
assert row['state']=='failed' and row['error_code']=='retry_exhausted' and j['attempt']==2
replay(j['id']);row,j=fetch(a['id']);assert process(event(j))=='ready'
report['retry']={'transitions':transitions,'seconds':round(time.monotonic()-started,3),'replayed_attempt':fetch(a['id'])[1]['attempt']}
report['invariants']['transient_backoff_exhaustion_then_replay']=True
# Real corrupted JPEG whose header is valid: heavy decoder fails safely in worker.
jpg=io.BytesIO();Image.new('RGB',(320,200),'blue').save(jpg,format='JPEG');truncated=jpg.getvalue()[:-100]
a,j=upload(truncated,'image/jpeg');process(event(j));bad,_=fetch(a['id'])
assert bad['state']=='failed' and bad['error_code']=='invalid_image'
report['invariants']['invalid_decode_permanent_failed']=True
# EXIF, alpha, small image and all on-disk variants checked by the real decoder.
transforms=[]
for kind in ['alpha','exif','small']:
out=io.BytesIO()
if kind=='alpha':
Image.new('RGBA',(20,10),(255,0,0,0)).save(out,format='PNG');mime='image/png'
elif kind=='exif':
image=Image.new('RGB',(200,320),'green');exif=Image.Exif();exif[274]=6;image.save(out,format='JPEG',exif=exif);mime='image/jpeg'
else:Image.new('RGB',(5,3),'yellow').save(out,format='PNG');mime='image/png'
a,j=upload(out.getvalue(),mime);assert process(event(j))=='ready'
r=c.call('GET','/api/v1/attachments/'+a['id']+'/content?variant=thumbnail',actor=bob)
with Image.open(io.BytesIO(r['bytes'])) as thumb:
expected=(20,10) if kind=='alpha' else (256,160) if kind=='exif' else (5,3)
assert thumb.size==expected
assert not thumb.getexif()
if kind=='alpha':assert min(thumb.getpixel((0,0)))>=250
transforms.append({'kind':kind,'size':thumb.size,'exif_removed':not bool(thumb.getexif())})
report['transforms']=transforms
pending,pending_job=upload()
dry=cleanup(False);fixed=cleanup(True);repeat=cleanup(True)
assert fetch(pending['id'])[0]['state']=='queued' and objects.head(pending_job['source_key']) is not None
report['invariants']['cleanup_preserves_current_and_fresh_failed']=True
assert repeat['counts']['ready_staging']==0
report['invariants']['ready_staging_cleanup_idempotent']=True
report['cleanup']=[dry,fixed,repeat]
print(json.dumps(report,indent=2))
if __name__=='__main__': main()
+28
View File
@@ -0,0 +1,28 @@
"""Figures from real accepted/ready and cgroup samples, no synthetic results."""
import json
from pathlib import Path
import statistics
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
r=Path(__file__).resolve().parents[1]/'evidence'
x=json.loads((r/'resources.json').read_text())['experiments']
fig,axes=plt.subplots(1,2,figsize=(11,4.3),layout='constrained')
labels=['320×200\n1,3 KiB','1800×1400\n7,21 MiB','10000×4000\n0,51 MiB']
a=[max(v['accepted_seconds'] for v in e['replies']) for e in x]
b=[max(v['ready_seconds'] for v in e['replies']) for e in x]
axes[0].bar([i-.18 for i in range(3)],a,.36,label='p95 до 202')
axes[0].bar([i+.18 for i in range(3)],b,.36,label='p95 до ready')
axes[0].set(xticks=range(3),xticklabels=labels,ylabel='Секунды',title='10 одновременных загрузок · worker=1')
axes[0].legend()
peak=[int(e['after']['memory.peak'])/1048576 for e in x]
anon=[max(int(dict(line.split() for line in s['worker']['memory.stat'].splitlines())['anon']) for s in e['samples'])/1048576 for e in x]
axes[1].bar([i-.18 for i in range(3)],peak,.36,label='cgroup memory.peak')
axes[1].bar([i+.18 for i in range(3)],anon,.36,label='Пик resident anon')
axes[1].axhline(768,color='#d62728',linestyle='--',label='Лимит worker')
axes[1].set(xticks=range(3),xticklabels=labels,ylabel='MiB',title='Worker + дочерний декодер')
axes[1].legend(fontsize=8)
for ax in axes:ax.grid(axis='y',alpha=.2);ax.set_axisbelow(True)
fig.suptitle('ЛР 5 · реальные измерения · concurrency=1, prefetch=1')
fig.savefig(r/'pipeline.png',dpi=150)
fig.savefig(r/'pipeline.svg')
+36
View File
@@ -0,0 +1,36 @@
"""Publication plot from measured cost/retention JSON; matplotlib is research-only."""
import json
from pathlib import Path
import statistics
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
ROOT=Path(__file__).resolve().parents[1]
data=json.loads((ROOT/'evidence/observability-cost.json').read_text())
modes=['off','full','sample10','sample25']
labels=['Off','Full 100%','Sample 10%','Sample 25%']
colors=['#64748b','#2563eb','#0d9488','#d97706']
fig,axs=plt.subplots(2,3,figsize=(16,9),layout='constrained')
def draw(ax,values,title,ylabel):
med=[statistics.median(v) for v in values]
low=[m-min(v) for m,v in zip(med,values)]
high=[max(v)-m for m,v in zip(med,values)]
ax.bar(range(4),med,color=colors,yerr=[low,high],capsize=4,alpha=.85)
for i,v in enumerate(values):ax.scatter([i]*len(v),v,color='#0f172a',s=14,zorder=3)
ax.set_xticks(range(4),labels,rotation=12)
ax.set_title(title,loc='left',fontweight='bold')
ax.set_ylabel(ylabel)
ax.grid(axis='y',alpha=.2)
ax.set_axisbelow(True)
rows=[[r for r in data['runs'] if r['mode']==mode] for mode in modes]
draw(axs[0,0],[[r['p95_seconds']*1000 for r in group] for group in rows],'HTTP p95','milliseconds')
draw(axs[0,1],[[r['resource']['api']['cpu_seconds'] for r in group] for group in rows],'API CPU (both replicas)','CPU seconds / run + drain')
draw(axs[0,2],[[r['collector_ingest']['otelcol_receiver_accepted_spans_total'] for r in group] for group in rows],'Accepted traces','spans / run + drain')
draw(axs[1,0],[[r['resource']['otel-collector']['rx_bytes']/1024 for r in group] for group in rows],'Collector network received','KiB (includes scrape / protocol traffic)')
draw(axs[1,1],[[r['resource']['api']['peak_sampled_anonymous_bytes']/1048576 for r in group] for group in rows],'API anonymous memory','MiB (sum of 1 Hz sampled replica peaks)')
draw(axs[1,2],[[r['resource']['worker']['cpu_seconds'] for r in group] for group in rows],'Worker CPU incl renderer','CPU seconds / run + drain')
fig.suptitle('Lab 6: cost of observability | 600 mixed requests per run (30 uploads) | 30 req/s | 3 repeats',fontsize=15,fontweight='bold')
fig.savefig(ROOT/'evidence/observability-cost.png',dpi=150)
fig.savefig(ROOT/'evidence/observability-cost.svg')
print('Plot written from actual per-run JSON; dots = repeats, bars = median, whiskers = min/max.')
+52
View File
@@ -0,0 +1,52 @@
"""Finite simultaneous jobs including 40 MP; record actual worker cgroup peak/RSS."""
import concurrent.futures
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
from client import Client
from images import png
def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip()
def memory(container):
code="import json;from pathlib import Path;p=Path('/sys/fs/cgroup');print(json.dumps({k:(p/k).read_text().strip() for k in ['memory.current','memory.peak','memory.events','memory.stat','cpu.stat']}))"
return json.loads(docker('exec',container,'python','-c',code))
def main():
c=Client(timeout=30);a,b,e,chat=c.bootstrap()
profiles=[(320,200,False),(1800,1400,True),(10000,4000,False)]
result=[]
for w,h,noise in profiles:
content=png(w,h,noise=noise)
docker('compose','restart','worker');container=docker('compose','ps','-q','worker')
before=memory(container);samples=[];done=threading.Event();start=time.monotonic()
def sample():
while not done.is_set():
samples.append({'seconds':round(time.monotonic()-start,4),'worker':memory(container)})
done.wait(.05)
sampler=threading.Thread(target=sample);sampler.start()
barrier=threading.Barrier(10)
def upload(i):
barrier.wait();t=time.monotonic()
r=Client(timeout=30).call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=load.png',202,actor=a,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
accepted=time.monotonic()-t
ready=Client(timeout=30).wait_ready(b,r['json'])
assert ready['sha256']==hashlib.sha256(content).hexdigest()
return {'index':i,'accepted_seconds':accepted,'ready_seconds':time.monotonic()-t,'status':ready['status']}
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: replies=list(pool.map(upload,range(10)))
finally:done.set();sampler.join()
after=memory(container);assert 'oom_kill 0' in after['memory.events']
result.append({'width':w,'height':h,'bytes':len(content),'clients':10,'before':before,'after':after,'samples':samples,'replies':replies,'seconds':time.monotonic()-start})
print(json.dumps({'method':'10 simultaneous uploads, worker concurrency=1; cgroup v2 samples include child renderer; anon is resident anonymous memory, memory.peak includes tmpfs/kernel/cache','experiments':result},indent=2))
if __name__=='__main__':main()
+38
View File
@@ -0,0 +1,38 @@
"""Separate synchronized rare-error/slow-success retention experiment."""
import json
from pathlib import Path
from observability_cost import set_mode, probes
from observability import save, trace, wait
import urllib.error
def verify_presence(rows):
for row in rows:
if row["trace_sampled"]:
data=wait(lambda:trace(row["trace_id"]),timeout=90)
assert data.get("batches",data.get("resourceSpans"))
row["tempo_trace_found"]=True
else:
try:
trace(row["trace_id"])
except urllib.error.HTTPError as error:
assert error.code==404
row["tempo_trace_found"]=False
else:
raise AssertionError("Unsampled trace unexpectedly stored")
return rows
def main():
result={'method':'Three repeats per full/10%/25% setting; each holds sessions lock until pg_locks confirms it. 3x503 plus one slow200 per repeat. Sampling is ordinary SDK head sampling without forced traceparent.','runs':[]}
try:
for mode,ratio in [('full',1),('sample10',.1),('sample25',.25)]:
c=set_mode(1,ratio)
actor=c.user('retention')
for repeat in range(3):
result['runs'].append({'mode':mode,'repeat':repeat,'probes':verify_presence(probes(c,actor,mode))})
save('sampling-retention.json',result)
print(mode,repeat,'retention recorded',flush=True)
finally:set_mode(1,.25)
print('PASS: synchronized rare 503/slow200 retention study; balanced sampling restored.')
if __name__=='__main__':main()
+29
View File
@@ -0,0 +1,29 @@
"""Private S3 and job envelopes do not expose source bytes or credentials."""
import json
from pathlib import Path
import sys
import urllib.error
import urllib.request
sys.path[:0]=['/app/src',str(Path.cwd()/'tests')]
from client import Client
from pipeline import connect,validate,event
from objects import Objects
from postgres import setting
c=Client();a,b,e,chat=c.bootstrap();att,raw=c.attachment(a,chat);ready=c.wait_ready(b,att)
with connect() as conn:
row=conn.execute('SELECT object_key, current_job_id FROM attachments WHERE 1=1 AND id = %s',(att['id'],)).fetchone()
job=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(row['current_job_id'],)).fetchone()
message=event(job);message['attempt']=max(1,job['attempt'])
assert validate(message)
required={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'}
assert required <= set(message) <= required | {'traceparent','tracestate'}
assert validate(message)
assert len(json.dumps(message))<1024
for path in ('/'+Objects(setting).bucket+'/'+row['object_key'],'/'+Objects(setting).bucket+'?list-type=2'):
try:urllib.request.urlopen('http://s3:8333'+path,timeout=2);raise AssertionError('anonymous access')
except urllib.error.HTTPError as error:assert error.code==403
for key in ['schema_version','pipeline_version','attempt']:
bad=dict(message);bad[key]=True;assert not validate(bad)
for body in [dict(message,extra='x'),dict(message,created_at='bad'),dict(message,traceparent='bad')]:assert not validate(body)
print(json.dumps({'anonymous_object_and_list_403':True,'job_fields':sorted(message),'job_bytes':len(json.dumps(message)),'no_source_bytes_or_credentials_in_envelope':True,'strict_invalid_envelope_checks':True},indent=2))
+32
View File
@@ -0,0 +1,32 @@
"""A continuously trickling client still reaches the absolute upload deadline."""
import json
from pathlib import Path
import select
import socket
import sys
import time
import uuid
from urllib.parse import urlsplit
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client
c = Client()
alice, bob, eve, chat = c.bootstrap()
target = urlsplit(c.base)
connection = socket.create_connection((target.hostname, target.port), timeout=3)
header = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=slow_' + uuid.uuid4().hex + '.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n'
started = time.monotonic()
connection.sendall(header.encode())
response = b''
while time.monotonic() - started < 20:
if select.select([connection], [], [], .15)[0]:
response += connection.recv(4096)
if b'\r\n\r\n' in response:
break
connection.sendall(b'x')
connection.close()
elapsed = time.monotonic() - started
assert response.startswith(b'HTTP/1.1 400') or response.startswith(b'HTTP/1.0 400'), response[:100]
assert 14 <= elapsed < 19
assert b'x-instance-id:' in response.lower() and b'x-request-id:' in response.lower()
print(json.dumps({'trickle_seconds': round(elapsed, 3), 'http_status': 400, 'absolute_deadline_enforced': True}))
+177
View File
@@ -0,0 +1,177 @@
"""Extra black-box tests, without changing the course's public tests."""
import concurrent.futures
from pathlib import Path
import sys
import unittest
from urllib.parse import quote
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client as PublicClient, LAB
class Client(PublicClient):
def call(self, method, path, *args, headers=None, **kwargs):
headers = dict(headers or {})
if method == 'POST' and path.endswith('/messages'):
headers.setdefault('Idempotency-Key', uuid.uuid4().hex)
return super().call(method, path, *args, headers=headers, **kwargs)
class APIContract(unittest.TestCase):
def setUp(self):
self.client = Client()
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
self.path = '/api/v1/chats/' + self.chat['id']
def test_malformed_json_and_extra_fields(self):
for raw in (b'{', b'[]', b'{"text":"x","text":"y"}', b'{"text":NaN}', b'{"text":"\\ud800"}'):
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'})
for body in ({'text': 'ok', 'sender_id': self.eve['id']}, {'text': True}, {'text': ' '}, {'text': ''}, {'text': None}):
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body=body)
deep = b'{"text":' + b'[' * 1200 + b'0' + b']' * 1200 + b'}'
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=deep, headers={'Content-Type': 'application/json'})
def test_unicode_limits(self):
result = self.client.message(self.alice, self.chat, '👩' * 4000)['json']
self.assertEqual(result['text'], '👩' * 4000)
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': '👩' * 4001})
def test_unicode_nul_round_trip(self):
import base64
value = 'A\x00B\\u0000👩'
registration = dict(self.alice['registration'], username='nul_' + uuid.uuid4().hex[:20], display_name=value)
user = self.client.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json']
self.assertEqual(user['display_name'], value)
credential = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json']
self.assertEqual(session['user'], user)
actor = {'id': user['id'], 'token': session['token']}
# Exercise the database path and then the Redis profile cache.
for _ in range(2):
self.assertEqual(self.client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'], user)
chat = self.client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': value, 'member_ids': [actor['id']]}, schema='Chat')['json']
self.assertEqual(chat['title'], value)
self.assertEqual(self.client.call('GET', '/api/v1/chats/' + chat['id'], actor=actor, schema='Chat')['json'], chat)
contents = [value, '\x00' * 4000, '👩' * 3999 + '\x00']
for content in contents:
self.assertEqual(self.client.message(actor, chat, content)['json']['text'], content)
history = self.client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor, schema='MessagePage')['json']
self.assertEqual([message['text'] for message in history['items']], contents)
def test_foreign_chat(self):
for suffix in ('', '/messages'):
self.client.call('GET', self.path + suffix, 404, actor=self.eve)
self.client.call('POST', self.path + '/messages', 404, actor=self.eve, body={'text': 'hidden'})
self.client.call('GET', '/api/v1/chats/' + str(uuid.uuid4()), 404, actor=self.alice)
def test_registration_race(self):
body = dict(self.alice['registration'], username='race_' + uuid.uuid4().hex[:20])
def register(_):
return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status']
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
statuses = list(executor.map(register, range(10)))
self.assertEqual(statuses.count(201), 1)
self.assertEqual(statuses.count(409), 9)
def test_invalid_chat_creation(self):
members = ([self.alice['id'], self.alice['id']], [self.bob['id']], [self.alice['id'], str(uuid.uuid4())], [], [True])
for member_ids in members:
self.client.call('POST', '/api/v1/chats', 400, actor=self.alice, body={'title': 'rollback', 'member_ids': member_ids})
def test_cursor_scope_and_validation(self):
for i in range(3):
self.client.message(self.alice, self.chat, str(i))
cursor = self.client.call('GET', self.path + '/messages?limit=1', actor=self.alice, schema='MessagePage')['json']['next_cursor']
other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'other', 'member_ids': [self.alice['id']]}, schema='Chat')['json']
self.client.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(cursor), 400, actor=self.alice)
for query in ('limit=0', 'limit=101', 'limit=' + '9' * 5000, 'limit=true', 'limit=1&limit=2', 'cursor=', 'cursor=bad', 'cursor=e30=', 'offset=1'):
self.client.call('GET', self.path + '/messages?' + query, 400, actor=self.alice)
def test_concurrent_messages_and_pagination(self):
def send(i):
return Client().message(self.alice if i % 2 else self.bob, self.chat, 'parallel-' + str(i))['json']
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
sent = list(executor.map(send, range(100)))
found = []
cursor = None
while True:
url = self.path + '/messages?limit=7' + ('&cursor=' + quote(cursor) if cursor else '')
page = self.client.call('GET', url, actor=self.alice, schema='MessagePage')['json']
found.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
break
self.assertEqual(len(found), 100)
self.assertEqual(len({v['id'] for v in found}), 100)
self.assertEqual({v['id'] for v in found}, {v['id'] for v in sent})
self.assertEqual(found, sorted(found, key=lambda v: (v['created_at'], v['id'])))
def test_idempotency_concurrent_replicas(self):
key = uuid.uuid4().hex
def send(_):
return Client().message(self.alice, self.chat, 'Один раз\x00👩', key=key)
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
replies = list(executor.map(send, range(32)))
self.assertGreaterEqual(len({v['headers']['X-Instance-Id'] for v in replies}), 2)
self.assertTrue(all(v['json'] == replies[0]['json'] for v in replies))
page = self.client.call('GET', self.path + '/messages', actor=self.bob, schema='MessagePage')['json']
self.assertEqual(page['items'], [replies[0]['json']])
self.client.call('POST', self.path + '/messages', 409, actor=self.alice, body={'text': 'different'}, headers={'Idempotency-Key': key})
self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=self.alice)
self.client.call('POST', self.path + '/messages', 401, actor=self.alice, body={'text': 'Один раз\x00👩'}, headers={'Idempotency-Key': key})
def test_protocol_error_headers(self):
self.client.call('TRACE', '/api/v1/users/me', 400, actor=self.alice)
def test_idempotency_scope(self):
key = uuid.uuid4().hex
one = self.client.message(self.alice, self.chat, 'same', key=key)['json']
two = self.client.message(self.bob, self.chat, 'same', key=key)['json']
other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'scope', 'member_ids': [self.alice['id']]}, schema='Chat')['json']
three = self.client.message(self.alice, other, 'same', key=key)['json']
self.assertEqual(len({one['id'], two['id'], three['id']}), 3)
def test_idempotency_semantic_body(self):
key = uuid.uuid4().hex
one = self.client.message(self.alice, self.chat, 'semantic', key=key)['json']
two = self.client.call('POST', self.path + '/messages', 201, actor=self.alice, body={'attachment_ids': [], 'text': 'semantic'}, headers={'Idempotency-Key': key}, schema='Message')['json']
self.assertEqual(one, two)
def test_idempotency_invalid_keys(self):
for key in (None, '', 'x' * 129, 'space key', 'bad,key', 'slash/key'):
headers = {} if key is None else {'Idempotency-Key': key}
PublicClient().call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': 'valid'}, headers=headers)
for raw in (b'{"text":true}', b'{"text":"ok","extra":1}', b'{"text":"ok","attachment_ids":false}'):
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'})
def test_idempotency_conflicting_race(self):
key = uuid.uuid4().hex
def send(i):
return Client().call('POST', self.path + '/messages', [201, 409], actor=self.alice, body={'text': str(i % 2)}, headers={'Idempotency-Key': key}, schema=None)
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
replies = list(executor.map(send, range(32)))
self.assertEqual(sum(v['status'] == 201 for v in replies), 16)
self.assertEqual(sum(v['status'] == 409 for v in replies), 16)
page = self.client.call('GET', self.path + '/messages', actor=self.alice, schema='MessagePage')['json']
self.assertEqual(len(page['items']), 1)
@unittest.skipUnless(LAB >= 2, 'sessions start in lab2')
def test_unicode_password_basic_and_logout(self):
import base64
registration = {'username': 'unicode_' + uuid.uuid4().hex[:16], 'display_name': 'Софья', 'password': '🔐' * 127 + ':'}
self.client.call('POST', '/api/v1/users', 201, body=registration)
encoded = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
headers = {'Authorization': 'Basic ' + encoded}
session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers=headers, schema='Session')['json']
actor = {'id': session['user']['id'], 'token': session['token']}
self.client.call('GET', '/api/v1/users/me', actor=actor)
self.client.call('GET', '/api/v1/users/me', 401, headers=headers)
for authorization in ('Basic !!!', 'Bearer wrong', '', 'Basic Zm9v'):
self.client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': authorization})
self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
self.client.call('GET', '/api/v1/users/me', 401, actor=actor)
if __name__ == '__main__':
unittest.main(verbosity=2)
+52
View File
@@ -0,0 +1,52 @@
"""Independent tests for async image metadata, transformations and access."""
import io
from pathlib import Path
import sys
import unittest
import uuid
from urllib.parse import quote
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'tests')]
from client import Client
from images import bomb
class Images(unittest.TestCase):
def setUp(self):
self.c=Client(); self.alice,self.bob,self.eve,self.chat=self.c.bootstrap()
self.path='/api/v1/chats/'+self.chat['id']
self.raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes()
def test_async_variants_and_replay(self):
queued,_=self.c.attachment(self.alice,self.chat)
self.assertEqual(queued['status'],'queued')
key=uuid.uuid4().hex
one=self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json']
self.assertEqual(self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json'],one)
ready=self.c.wait_ready(self.alice,queued)
self.assertEqual({v['name'] for v in ready['variants']},{'thumbnail','crop','optimized'})
dimensions={v['name']:(v['width'],v['height']) for v in ready['variants']}
self.assertEqual(dimensions,{'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)})
for v in ready['variants']:
import hashlib
r=self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant='+v['name'],actor=self.bob)
self.assertEqual(hashlib.sha256(r['bytes']).hexdigest(),v['sha256'])
self.assertEqual(len(r['bytes']),v['size_bytes'])
self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant=absent',404,actor=self.bob)
def test_mime_bytes_pixels_and_filename(self):
path=self.path+'/attachments?filename=test.png'
for raw,mime,status in [(b'not image','image/png',415),(self.raw[:20],'image/png',415),(self.raw,'image/jpeg',415),(bomb(),'image/png',413),(b'x'*(10485760+1),'image/png',413)]:
self.c.call('POST',path,status,actor=self.alice,raw=raw,headers={'Content-Type':mime})
for name in ('../x.png','a\\x.png','a\0.png','a\n.png','a'*256,''):
self.c.call('POST',self.path+'/attachments?filename='+quote(name,safe=''),400,actor=self.alice,raw=self.raw,headers={'Content-Type':'image/png'})
def test_private_and_cross_chat(self):
att,_=self.c.attachment(self.alice,self.chat)
for suffix in ('','/content'):
self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=self.eve)
self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,401)
other=self.c.call('POST','/api/v1/chats',201,actor=self.alice,body={'title':'other','member_ids':[self.alice['id']]},schema='Chat')['json']
self.c.call('POST','/api/v1/chats/'+other['id']+'/messages',404,actor=self.alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
if __name__=='__main__': unittest.main(verbosity=2)