ЛР 6 — БВТ2301 — Майорова — на 5 #52
@@ -18,3 +18,5 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
build
|
||||
|
||||
+16
-27
@@ -1,30 +1,19 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
APP_PORT=18093
|
||||
GRAFANA_PORT=13006
|
||||
PROMETHEUS_PORT=19096
|
||||
LOKI_PORT=13106
|
||||
TEMPO_PORT=13206
|
||||
COLLECTOR_PORT=18896
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
API_MEMORY=256m
|
||||
HTTP_WORKERS=16
|
||||
DB_POOL_MAX=8
|
||||
DB_ACCESS_MODE=optimized
|
||||
WORKER_CONCURRENCY=1
|
||||
MAX_PENDING_JOBS=100
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
JOB_TIMEOUT_SECONDS=20
|
||||
JOB_LEASE_SECONDS=60
|
||||
TELEMETRY_ENABLED=1
|
||||
TRACE_SAMPLE_RATIO=0.25
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
name: Application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
BASE_URL: http://127.0.0.1:18093
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: python3 scripts/bootstrap.py
|
||||
- run: docker compose build api
|
||||
- run: docker compose up -d --wait
|
||||
- run: python3 scripts/wait_ready.py
|
||||
- run: make check test test-student BASE_URL=$BASE_URL
|
||||
- run: make test-observability BASE_URL=$BASE_URL
|
||||
- run: make test-persistence BASE_URL=$BASE_URL ACTION_SCRIPT=scripts/restart.sh
|
||||
- if: always()
|
||||
run: docker compose down -v
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 MALLOC_ARENA_MAX=2
|
||||
WORKDIR /app
|
||||
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY src/ ./src/
|
||||
COPY migrations/ ./migrations/
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"]
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "src/server.py"]
|
||||
@@ -28,3 +28,19 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: test-student
|
||||
test-student:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p 'test_*.py' -v
|
||||
|
||||
.PHONY: test-observability test-incidents test-telemetry-outage
|
||||
test-observability:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py proof
|
||||
test-incidents:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py incidents
|
||||
test-telemetry-outage:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability.py backend
|
||||
|
||||
.PHONY: research
|
||||
research:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/observability_cost.py
|
||||
|
||||
@@ -1,37 +1,350 @@
|
||||
# Отчёт — лабораторная 6
|
||||
# Лабораторная 6 — наблюдаемость мессенджера
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
**Майорова Софья, БВТ2301. Целевая оценка: 5, трек 1 «Цена наблюдаемости».**
|
||||
Дата выполнения: 20 ноября 2026 года. Оценку подтверждает преподаватель.
|
||||
|
||||
Продолжена [собственная lab5, PR 7](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/7).
|
||||
Перенесены только исходники, миграции, свои тесты и необходимые конфигурации.
|
||||
README/COURSE/contracts/tests текущего skeleton lab6 сохранены, assertions публичных проверок не менялись.
|
||||
Трек 2 с multi-window burn rate не заявляется.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Docker Engine/Compose v2, Linux-контейнеры, Python 3.9+ для host-проверок.
|
||||
Сервер и worker Python 3.12.8 внутри Docker. На этом Mac:
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
colima start --profile mtusi-labs
|
||||
export DOCKER_CONTEXT=colima-mtusi-labs
|
||||
cd "/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab6"
|
||||
python3 scripts/bootstrap.py
|
||||
docker compose build api
|
||||
docker compose up -d --wait
|
||||
export BASE_URL=http://127.0.0.1:18093
|
||||
python3 scripts/wait_ready.py
|
||||
make check test test-student BASE_URL=$BASE_URL
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
На другой машине клонировать публичный fork, выбрать solution/BVT2301-Mayorova и выполнять из корня clone.
|
||||
Образ приложения сначала строится через build api: worker, dispatcher, migration и audit используют тот же image.
|
||||
Bootstrap генерирует локальные случайные secrets, сохраняет существующие значения и .env.
|
||||
Каталог secrets 0700; полный S3 identity config 0600, монтируется только root gateway.
|
||||
Пароли, ключи, .env, volumes, backups и build не коммитятся и не попадают в build context.
|
||||
|
||||
## Доказательства критериев
|
||||
API: http://127.0.0.1:18093.
|
||||
Grafana: http://127.0.0.1:13006/d/messenger-lab6 — автоматический Viewer на localhost.
|
||||
GF_USERS_VIEWERS_CAN_EDIT разрешает Explore и временные правки запросов без сохранения dashboard;
|
||||
роль остаётся Viewer (настройка deprecated в Grafana 12.1.1, доступна в закреплённой версии).
|
||||
Импорт dashboard и трёх datasources выполняет provisioning из Git; ручной настройки не требуется.
|
||||
Prometheus http://127.0.0.1:19096, Loki 13106, Tempo 13206, собственные метрики Collector 18896.
|
||||
OTLP 4318 не публикуется на host. PostgreSQL/S3/broker/API-реплики тоже не публикуются в обход LB.
|
||||
Операторские UI присоединены к edge для корректного loopback port mapping Docker; backend и storage — внутренние сети.
|
||||
Наружу в Интернет эти порты не открыты. Grafana не является административной панелью бизнес-данных.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Остановка с сохранением истории: docker compose down.
|
||||
Не удалять volumes в проверке restart/failover. down -v предназначен только для намеренно уничтожаемого отдельного тестового стенда.
|
||||
На этой машине старый Docker proxy не разрешался DNS.
|
||||
После отдельного разрешения пользователя сохранён backup /etc/docker/daemon.json.lab6-proxy-backup внутри VM,
|
||||
удалена неработающая proxy-секция и перезапущен Docker daemon. Реестр проверен по прямому HTTPS.
|
||||
Это локальная настройка окружения, не часть приложения и не обязательный шаг для другого clone.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Архитектура
|
||||
|
||||
## Эксперименты и отказы
|
||||
```text
|
||||
Клиент → HAProxy → API ×2 → PostgreSQL
|
||||
├→ Redis cache профилей
|
||||
└→ приватный S3 staging
|
||||
PostgreSQL jobs/outbox → dispatcher → Redis Streams/AOF → worker → S3 results + DB ready
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
API / dispatcher / worker → OpenTelemetry SDK → OTLP HTTP/protobuf → Collector
|
||||
├→ metrics → Prometheus → Grafana
|
||||
├→ logs → Loki → Grafana Explore
|
||||
└→ traces → Tempo → Grafana Explore
|
||||
```
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Prometheus хранит временные ряды, Loki — структурные логи, Tempo — трассы.
|
||||
Grafana выполняет запросы и визуализирует; Collector принимает/обрабатывает/экспортирует и не заменяет долговременные хранилища.
|
||||
PostgreSQL остаётся источником истины бизнес-данных и сессий. Наблюдаемость не участвует в COMMIT подтверждённого сообщения/задания.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Сохранён минимум предыдущих работ: scrypt и сессии с expiry/logout, Unicode/U+0000, ACL чатов,
|
||||
keyset, атомарный Idempotency-Key на 24h, две реплики и migration job, Redis profile cache,
|
||||
приватные изображения, durable staging/outbox, bounded renderer, at-least-once/retry/lease,
|
||||
immutable versioned results, version fencing, failed registry/replay и операторский cleanup.
|
||||
API возвращает 202 queued после S3 staging и COMMIT; resize выполняет отдельный worker.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Миграции 001–006. Новая 006 добавляет trace_context JSONB к attachment_jobs.
|
||||
Миграция выполняется отдельным service под advisory lock; API проверяет наличие 006 до обслуживания.
|
||||
Старые jobs получают пустой carrier и продолжают работать; бизнес-API не расширялся.
|
||||
|
||||
## Версии и ресурсы
|
||||
|
||||
Все Docker images закреплены version + digest в Compose/Dockerfile.
|
||||
Python 3.12.8; PostgreSQL 17.2; Redis 7.4.1; SeaweedFS 4.51; HAProxy 3.0.5.
|
||||
OTel Python API/SDK/HTTP exporter 1.36.0, semantic conventions 0.57b0;
|
||||
Collector contrib 0.132.0; Prometheus 3.5.0; Loki 3.5.3; Tempo 2.8.2; Grafana 12.1.1.
|
||||
Все 27 Python distributions, включая транзитивные, закреплены в requirements.txt по успешной сборке.
|
||||
|
||||
API ×2: по 1 CPU/256MiB; worker 1 CPU/768MiB, concurrency1; dispatcher 1 CPU/256MiB.
|
||||
PostgreSQL 1 CPU/512MiB; broker0.5CPU/192MiB/AOF always; Redis cache0.25CPU/64MiB.
|
||||
S3 store1CPU/512MiB и gateway0.5CPU/512MiB; LB0.5CPU/64MiB.
|
||||
Collector/Prometheus/Loki/Grafana — по 0.5CPU/256MiB, Tempo0.5CPU/512MiB.
|
||||
Сумма CPU-лимитов — overcommit, не выделенные физические ядра.
|
||||
Окружение: macOS arm64, Colima Linux VM4vCPU/около4GiB; локальный synthetic workload.
|
||||
|
||||
## Схема сигналов и единицы
|
||||
|
||||
Ручная SDK-инструментация ограничивает атрибуты разрешённым набором.
|
||||
Resource: service.name=messenger-api / messenger-worker / messenger-dispatcher;
|
||||
service.version=lab6.1; service.instance.id — случайный alias процесса;
|
||||
deployment.environment.name=lab. Дополнительно SDK сообщает собственную версию/язык.
|
||||
|
||||
| Реальное имя Prometheus | Значение |
|
||||
| --- | --- |
|
||||
| http_server_requests_total | Обработанные business requests; method, route template, status_class |
|
||||
| http_server_duration_seconds_bucket/count/sum | Полная обработка HTTP, включая отправку ответа |
|
||||
| jobs_processed_total | Попытки worker: ready, failed_or_retry, noop, fenced, error; job_type=image |
|
||||
| job_processing_duration_seconds_bucket/count/sum | Время попытки, тот же outcome |
|
||||
| job_ready_duration_seconds_bucket/count/sum | От created_at durable job перед COMMIT до публикации ready |
|
||||
| job_retries_total | Решения retry/terminal failed, outcome=retry или failed |
|
||||
| queue_depth | Незавершённые queued/processing в PostgreSQL, включая outbox до broker |
|
||||
| queue_oldest_age_seconds | Возраст самой старой такой job; backlog не processing latency |
|
||||
| queue_observation_ok | 0 при невозможности наблюдать PostgreSQL backlog |
|
||||
| db_pool_used/waiting/capacity | Используемые/ожидающие/максимум соединений API |
|
||||
| worker_busy/capacity | Активные попытки/число slots |
|
||||
| process_cpu_seconds_total | CPU процесса, cumulative |
|
||||
| process_resident_memory_bytes | RSS процесса через /proc/self/statm |
|
||||
| container_memory_bytes | Cgroup memory.current, включая renderer и cache |
|
||||
| spans_finished/exported/export_failed | Число sampled SDK spans и результат отправки |
|
||||
|
||||
В gauge количества unit пустой: иначе unit1 экспортируется Prometheus как _ratio.
|
||||
Очередь именуется постоянным attachment-jobs; отдельного динамического queue label не нужно.
|
||||
Service instance — resource attribute с ограниченной жизнью процесса; churn реплик всё равно создаёт новые series.
|
||||
Health полностью исключён из business metrics/logs/traces. Не распознанные маршруты имеют template unmatched.
|
||||
user/chat/message/attachment/request/job/trace IDs, filename, raw URL и текст не являются labels метрик.
|
||||
|
||||
Границы duration histogram, секунды:
|
||||
0.001,0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2,5,10,20,30,60,+Inf.
|
||||
Dashboard агрегирует buckets всех реплик:
|
||||
histogram_quantile(0.95, sum by(le)(rate(http_server_duration_seconds_bucket[1m]))).
|
||||
p50/p99 построены аналогично; средние p95 реплик не усредняются.
|
||||
Точность percentile ограничена границами buckets и количеством наблюдений.
|
||||
|
||||
## Трассы, логи и очередь
|
||||
|
||||
Handler создаёт SERVER span с method/route/status без headers/body/query.
|
||||
Connection context создаёт CLIENT db.transaction span, включающий ожидание пула, SQL и завершение транзакции,
|
||||
но без текста SQL, параметров, connection string или сырого текста exception.
|
||||
Это время unit of work, а не чистый server-side SQL execution time: S3 PUT под transaction виден дочерним S3 span.
|
||||
S3 get span заканчивается при получении SDK response; потоковое чтение Body входит во время родительского HTTP/worker span.
|
||||
S3 wrapper записывает только service S3 и имя операции; ключи, filenames, URLs и credentials не записываются.
|
||||
Worker consumer span включает processing и реальные дочерние S3/DB spans.
|
||||
Внутри изолированного renderer отдельный SDK не запускается.
|
||||
|
||||
W3C traceparent/tracestate сохраняются с job в той же транзакции outbox.
|
||||
Dispatcher извлекает durable context, создаёт PRODUCER span и вкладывает его carrier в envelope.
|
||||
Worker извлекает envelope и создаёт CONSUMER child producer. Retry публикуется с тем же исходным trace,
|
||||
новым producer/consumer и номером попытки. Envelope содержит ID/версию/попытку/carrier, а не bytes изображения.
|
||||
|
||||
Структурный JSON log содержит timestamp, severity, service.name, event, trace_id, span_id, sampled;
|
||||
HTTP — X-Request-Id, template/method/status/duration; job — технический job_id/attempt/outcome.
|
||||
Эти технические ID нужны для сквозной диагностики синтетического задания и не индексируются как metric labels.
|
||||
LoggingHandler отправляет OTLP logs; Loki принимает native OTLP и хранит остальные поля как structured metadata/body.
|
||||
Grafana derived field TraceID ведёт из Loki в Tempo; tracesToLogs фильтрует тот же trace_id.
|
||||
Имена полей/шаблоны фиксированы. Тела сообщений, картинки, пароли, Authorization, tokens и signed URLs исключены.
|
||||
|
||||
Проверка proof выполняет реальный upload с известным traceparent, ждёт ready,
|
||||
проверяет родителя consumer=producer и цепочку предков producer до SERVER upload,
|
||||
проверяет worker S3/DB spans и логи всех стадий, связывает response X-Request-Id.
|
||||
Отдельно остановлен worker, принят файл, приостановлен реальный S3 gateway, worker запущен:
|
||||
наблюдается S3 timeout/retry; после возврата S3 второй consumer достигает ready в той же трассе.
|
||||
Canary message/password/token/username/filename не найдены в проверяемом trace и app logs с начала controlled proof (лимит 5000);
|
||||
это ограниченная динамическая проверка плюс allowlist кода, не математическое доказательство отсутствия любых утечек.
|
||||
Метрики проверены на запрещённые ID labels.
|
||||
|
||||
## SLI/SLO и алерты
|
||||
|
||||
Полные определения/ограничения — infra/RUNBOOK.md.
|
||||
Окно5мин: API availability>=99% обработанных business requests без5xx; health исключён,
|
||||
4xx не смешиваются с отказами сервера. Отказ LB/сети до API данным SLI не покрывается.
|
||||
Latency SLO: >=95% завершённых ready jobs укладываются в5сек от durable job creation.
|
||||
Это условный latency SLO, не доступность pipeline; queued/failed отдельно видны через backlog/outcomes/alerts.
|
||||
|
||||
Prometheus правила из infra/alerts.yaml, evaluation2s:
|
||||
- APIHighErrorRate: 5xx share за30s>5%, for6s.
|
||||
- QueueLag: oldest>10s и depth>0, for6s.
|
||||
- QueueObservationUnavailable: observer0, for6s.
|
||||
Это короткие учебные operational alerts, не трек2 multi-window burn rate.
|
||||
Нет email/Slack или внешней доставки.
|
||||
|
||||
## Два инцидента и диагностика
|
||||
|
||||
1. Worker остановлен, API принял3 изображения.
|
||||
Queue depth/oldest растут, завершения worker прекращаются; upload202 и логи acceptance/publish продолжаются.
|
||||
QueueLag firing примерно через22.77сек. После возврата worker все три ready, backlog0 и alert resolved.
|
||||
Значения gauges остановленного процесса могут оставаться в scrape cache до30s: это явно учитывается в runbook.
|
||||
|
||||
2. Реальная ACCESS EXCLUSIVE блокировка sessions удерживается35s.
|
||||
Readiness SELECT1 остаётся зелёной, защищённый endpoint ждёт SQL и получает503 по lock_timeout1.5s.
|
||||
APIHighErrorRate firing примерно через13.14сек. В сохранённой трассе db.transaction — долгий/error span;
|
||||
это отделяет ожидание PostgreSQL от HTTP handler/resize. После COMMIT блокировки прежний Bearer снова работает,
|
||||
30s error window очищается и alert resolved.
|
||||
|
||||
Сырые timestamps, firing snapshots, запросы метрик, error latencies и DB trace — evidence/incidents.json.
|
||||
Разбор опирается на наблюдения; способ инъекции задокументирован отдельно и не заменяет trace/metric evidence.
|
||||
|
||||
## Отказ telemetry backend и конечная буферизация
|
||||
|
||||
Tempo остановлен примерно на56сек. За это время API подтвердил135 сообщений и5 файлов,
|
||||
worker завершил файлы, все подтверждённые сообщения найдены в истории.
|
||||
Collector memory snapshots: примерно58.21–62.05MiB при лимите256MiB; наблюдены156 failed exported spans.
|
||||
После восстановления найден новый реальный trace. Потерянная telemetry не пересоздаётся из бизнес-данных.
|
||||
|
||||
SDK: BatchSpanProcessor queue512/batch128, logs queue256/batch64, schedule500ms,
|
||||
HTTP timeout1s; metric export каждые2s. Отправка асинхронная, очереди ограничены.
|
||||
Collector memory_limiter limit192/spike32MiB, batch<=256;
|
||||
два exporter queue по128 batches,2consumers, retry budget10s.
|
||||
Prometheus exporter имеет expiration30s. Нет безразмерной очереди или бесконечного retry.
|
||||
Лимит batches не является точным лимитом байт; SDK/receiver имеют дополнительные ограничения,
|
||||
container memory лимит — последняя граница. Локальный тест конечной длительности не доказывает поведение при любом трафике.
|
||||
|
||||
Метрики хранятся24h и ограничены256MB TSDB; Loki/Tempo retention24h; Grafana SQLite/provisioning.
|
||||
Именованные metrics_data/logs_data/traces_data/grafana_data сохраняются при restart.
|
||||
Loki/Tempo retention выполняется асинхронно и не является жёсткой дисковой квотой.
|
||||
|
||||
## Исследование цены наблюдаемости
|
||||
|
||||
Гипотеза: SDK export/sampling влияет на задержки, CPU/RAM и ingest; уменьшение доли трасс сокращает ingest,
|
||||
но часть редких ошибок и медленных запросов не имеет полного trace.
|
||||
Методика student-tests/observability_cost.py:
|
||||
off, full1.0, head sampling0.10 и0.25; три повтора, порядок режимов циклически сдвигается.
|
||||
Одинаковые лимиты,2API,worker concurrency1, fixture320x200.
|
||||
Каждый прогон: фиксированные30 scheduled requests/s20s,600 запросов:
|
||||
5% upload,15% messages,80% profile reads;16client threads.
|
||||
Exporter off оставляет установленные SDK/hooks, но не создаёт providers и не экспортирует signals.
|
||||
Это не сравнение с переписанной версией без SDK imports.
|
||||
|
||||
Измеряются клиентские p50/p95/p99, подтверждённый RPS/ошибки,
|
||||
CPU cgroup API/worker/dispatcher/Collector/хранилищ,
|
||||
anonymous/working-set memory samples1Hz,
|
||||
точные counter deltas принятых spans/logs/metric points и Collector receive network bytes.
|
||||
Последние включают OTLP, Prometheus scrape и backend replies: это не чистый размер protobuf payload.
|
||||
CPU/ingest включают ожидание exporter drain5s и проверки ready после основного потока.
|
||||
История БД/telemetry сохраняется между режимами; новые API/worker/dispatcher создаются на каждом режиме.
|
||||
Пять секунд прогрева/drain, фиксированная среда и смена порядка уменьшают, но не исключают шум/влияние кэша.
|
||||
UUID/sampling стохастические; нагрузка локальная и короткая. Это не capacity ceiling и не production SLA.
|
||||
|
||||
Первоначальные SQL-lock probes после измерительных окон не синхронизировали момент HTTP с pg_locks:
|
||||
часть блокировок успевала завершиться при старте docker exec. Их фактические HTTP статусы сохранены
|
||||
в rare_probes основного raw JSON; выдавать их за гарантированные 503 нельзя.
|
||||
Для вывода о редких событиях выполнено отдельное исследование student-tests/sampling_retention.py:
|
||||
full/10%/25%, три повтора; в каждом три настоящих503 и один медленный200.
|
||||
Блокировки3.5s/1.2s проверяются через granted pg_locks перед HTTP, status и duration проверяются assertions.
|
||||
У каждого режима получено9ошибок и3медленных успешных запроса.
|
||||
|
||||
### Измеренные результаты
|
||||
|
||||
Всего12основных прогонов,7200/7200 подтверждённых запросов без ошибок,360/360 изображений ready.
|
||||
Ниже медиана трёх прогонов; ranges и каждый raw result доступны в JSON, точки/отрезки на графике.
|
||||
CPU — накопленные секунды двух API или worker за измерительное окно с drain, не проценты host CPU.
|
||||
|
||||
| Режим | HTTP p95, ms | p99, ms | API CPU, s | Worker CPU, s | Collector CPU, s | Принятые spans | Collector RX, KiB |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| off | 9.818 | 15.301 | 2.371 | 5.464 | 0.054 | 0 | 13.2 |
|
||||
| full | 9.695 | 14.553 | 2.799 | 5.533 | 0.194 | 1950 | 1067.0 |
|
||||
| 10% | 9.149 | 13.731 | 2.541 | 5.270 | 0.167 | 176 | 720.9 |
|
||||
| 25% | 10.557 | 15.699 | 2.618 | 5.220 | 0.183 | 467 | 802.8 |
|
||||
|
||||
| Режим | Сумма sampled anon peaks двух API, MiB | Collector anon peak, MiB | Logs / metric points, медиана |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| off | 159.60 | 56.02 | 0 / 0 |
|
||||
| full | 147.90 | 56.03 | 840 / 1011 |
|
||||
| 10% | 168.66 | 58.78 | 840 / 987 |
|
||||
| 25% | 147.82 | 58.72 | 840 / 1011 |
|
||||
|
||||
Сумма отдельных sampled peaks не является одновременным peak всей группы; интервалы memory/p95 перекрываются.
|
||||
Результаты не доказывают ускорение запросов или экономию RAM. Разница CPU небольшая и подвержена шуму.
|
||||
Full даёт примерно18% больше API CPU, чем off, в медианах этого опыта.
|
||||
25% уменьшает принятые spans на76.1%, сетевой RX Collector на24.8% и API CPU примерно на6.5% относительно full.
|
||||
Ingest логов/метрик сохраняется независимо от trace sampling, поэтому суммарный трафик не падает на75%.
|
||||
Head sampling применяется к целой цепочке: число дочерних spans в sampled image jobs меняет фактическую долю spans.
|
||||
После drain sampled spans SDK и accepted spans Collector совпали во всех12прогонах:
|
||||
дополнительная потеря между SDK и Collector в данном измерении0; несэмплированные spans в это число не входят.
|
||||
|
||||
| Режим | Сохранён trace ошибок503 | Сохранён trace медленных200 |
|
||||
| --- | ---: | ---: |
|
||||
| full | 9/9 | 3/3 |
|
||||
| 10% | 0/9 | 1/3 |
|
||||
| 25% | 1/9 | 1/3 |
|
||||
|
||||
Это наблюдения малой выборки, а не оценка вероятности или гарантия. Логи этих HTTP и полные метрики
|
||||
сохраняются; sampled проверяется в коррелированном HTTP log;
|
||||
для всех36probes также проверено реальное наличие sampled trace и404 для unsampled через Tempo API. Head sampling принимает решение до результата,
|
||||
поэтому не может гарантировать trace ошибки. Для расследования воспроизводимого сбоя временно выбирать1.0
|
||||
или отправлять контролируемый sampled W3C parent (как proof); это не автоматическое сохранение всех ошибок.
|
||||
|
||||
По умолчанию выбран25% как компромисс объёма трасс и покрытия при сохранённых logs/metrics.
|
||||
10% даёт ещё меньший ingest, но более редкие полные причинные цепочки. В режиме incident диагностики подходит full.
|
||||
Отдельный повтор выбранного режима:600/600 запросов,30/30 изображений ready,
|
||||
p95=10.587ms,p99=17.222ms,456accepted spans, дополнительная потеря SDK→Collector0.
|
||||
Результат — evidence/balanced-confirmation.json.
|
||||
Сырые результаты: [observability-cost.json](evidence/observability-cost.json),
|
||||
[cost-summary.json](evidence/cost-summary.json), [sampling-retention.json](evidence/sampling-retention.json).
|
||||
|
||||

|
||||
|
||||
## Проверки
|
||||
|
||||
Локально выполнено:
|
||||
- make check и полная OpenAPI3.1 validation — PASS, оригинальные contracts/tests без изменений.
|
||||
- make test и make test-student — public smoke и18тестов API/изображений PASS.
|
||||
- make test-observability — реальная causal trace, retry, correlated logs, low-cardinality и canary checks PASS.
|
||||
- make test-incidents — оба алерта firing/resolved, trace SQL lock и восстановление PASS.
|
||||
- make test-telemetry-outage — бизнес-записи/worker при отказе Tempo, bounded memory, новые traces после возврата PASS.
|
||||
- make test-persistence ACTION_SCRIPT=scripts/restart.sh — confirmed business data и старый Bearer сохранены;
|
||||
дополнительный persistence_telemetry.py подтвердил прежние spans/logs и historical Prometheus sample.
|
||||
- make test-failover ACTION_SCRIPT=scripts/stop-one.sh —120записей,1временная ошибка,
|
||||
первый успех после отказа2.602s; API восстановлен.
|
||||
- student-tests/security.py — anonymous object/list403, strict envelope PASS.
|
||||
- student-tests/pipeline_checks.py —13групп инвариантов, реальные exit77, lease recovery/version fencing/replay/cleanup PASS.
|
||||
- make research —12прогонов; отдельные sampling_retention.py —9синхронизированных прогонов.
|
||||
- promtool check rules —3правила PASS.
|
||||
- Grafana UI: dashboard с реальными рядами, Tempo→Loki по тому же trace_id и Loki TraceID→Tempo проверены.
|
||||
Скриншоты evidence/grafana.jpg, trace-ui.jpg, correlated-ui.jpg.
|
||||
|
||||
Настоящий отдельный clone commit0e78b2b с новыми secrets и пустыми volumes:
|
||||
сборка, миграции, public smoke,18тестов, causal/retry proof и полный restart с прежними
|
||||
business data/Bearer/Tempo/Loki/Prometheus history — PASS за83.54s.
|
||||
Тестовый Compose project mtusi-lab6-clean-6df218fa и только его volumes удалены;
|
||||
основной mtusi-lab6 сохранён. Evidence: clean-clone.json / clean-clone.txt.
|
||||
Изменения после clone ограничены отчётом, evidence и дополнительной проверкой исследования, бизнес-код не менялся.
|
||||
|
||||
Команды для повторения fault/research проверок запускать последовательно на отдельном учебном стенде:
|
||||
они управляют только текущим Compose project и временно останавливают его компоненты.
|
||||
CI application workflow добавлен; результат hosted CI не подтверждён, требующее maintainer approval выполнение не выдаётся за PASS.
|
||||
Сохранённые txt/JSON — фактические локальные результаты. Raw telemetry содержит только синтетические технические ID.
|
||||
|
||||
## Ограничения
|
||||
|
||||
Один host, один PostgreSQL/S3/broker/LB/Collector/backend каждого типа остаются SPOF.
|
||||
Доказывается отказ API-процесса, перезапуск с сохранением volumes и ограниченный отказ telemetry backend,
|
||||
а не потеря диска/питания, distributed HA или exactly-once.
|
||||
Миграции и outbox определяют корректность данных независимо от наличия telemetry.
|
||||
Логи и метрики тоже имеют bounded asynchronous exporters; при перегрузке/отказе Collector возможны их потери.
|
||||
Head sampling не гарантирует сохранение ошибки или редкого хвоста; для этого потребовалась бы отдельная политика tail sampling
|
||||
с доказанным буфером/памятью, которую мы не заявляем реализованной.
|
||||
Retention24h и file-based stores рассчитаны на локальный synthetic стенд, не промышленное хранение.
|
||||
|
||||
## Источники и использование ИИ
|
||||
|
||||
Собственная lab5, skeleton/contracts/public tests lab6.
|
||||
Официальные материалы:
|
||||
- https://opentelemetry.io/docs/languages/python/instrumentation/
|
||||
- https://opentelemetry.io/docs/collector/configuration/
|
||||
- https://opentelemetry.io/docs/concepts/context-propagation/
|
||||
- https://grafana.com/docs/loki/latest/send-data/otel/
|
||||
- https://grafana.com/docs/grafana/latest/administration/provisioning/
|
||||
- https://prometheus.io/docs/practices/histograms/
|
||||
|
||||
Codex использован для реализации, проверки, controlled incidents, исследования и отчёта.
|
||||
Численные результаты относятся к сохранённым локальным прогонам. Hosted CI не объявляется прошедшим без отдельного подтверждения.
|
||||
\nНастройка Viewer/Explore: https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#viewers_can_edit\n
|
||||
+298
@@ -0,0 +1,298 @@
|
||||
name: mtusi-lab6
|
||||
x-application: &application
|
||||
image: mtusi-messenger:lab6
|
||||
networks: [backend]
|
||||
environment: &environment
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
HTTP_WORKERS: ${HTTP_WORKERS:-16}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-8}
|
||||
DB_ACCESS_MODE: ${DB_ACCESS_MODE:-optimized}
|
||||
LOG_LEVEL: WARNING
|
||||
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
|
||||
TRACE_SAMPLE_RATIO: ${TRACE_SAMPLE_RATIO:-0.25}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_key
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret
|
||||
secrets: [database_url, cache_url, s3_api_key, s3_api_secret]
|
||||
tmpfs: ["/tmp:size=128m,mode=1777"]
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
cpus: ${API_CPUS:-1}
|
||||
stop_grace_period: 10s
|
||||
services:
|
||||
migrate:
|
||||
<<: *application
|
||||
command: [python, src/migrate.py]
|
||||
healthcheck:
|
||||
disable: true
|
||||
deploy:
|
||||
replicas: 1
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
api:
|
||||
<<: *application
|
||||
build: .
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
depends_on:
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
bucket-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
bucket-bootstrap:
|
||||
<<: *application
|
||||
command: [python, src/bucket_bootstrap.py]
|
||||
environment:
|
||||
<<: *environment
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret
|
||||
secrets: [s3_admin_key, s3_admin_secret]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
audit:
|
||||
<<: *application
|
||||
profiles: [tools]
|
||||
environment:
|
||||
<<: *environment
|
||||
BASE_URL: http://lb:8080
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_LEASE_SECONDS: 5
|
||||
secrets: [database_url, cache_url, s3_api_key, s3_api_secret, s3_admin_key, s3_admin_secret, s3_worker_key, s3_worker_secret, broker_url]
|
||||
healthcheck:
|
||||
disable: true
|
||||
deploy:
|
||||
replicas: 1
|
||||
dispatcher:
|
||||
<<: *application
|
||||
command: [python, src/pipeline.py, dispatch]
|
||||
environment:
|
||||
<<: *environment
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60}
|
||||
secrets: [database_url, broker_url]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
worker:
|
||||
<<: *application
|
||||
command: [python, src/pipeline.py, work]
|
||||
environment:
|
||||
<<: *environment
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_key
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
|
||||
secrets: [database_url, broker_url, s3_worker_key, s3_worker_secret]
|
||||
mem_limit: ${WORKER_MEMORY:-768m}
|
||||
tmpfs: ["/tmp:size=128m,mode=1777"]
|
||||
cpus: 1
|
||||
deploy:
|
||||
replicas: 1
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
bucket-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
broker:
|
||||
image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47
|
||||
command: [redis-server, --appendonly, "yes", --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction, --save, ""]
|
||||
volumes: [broker_data:/data]
|
||||
networks: [backend]
|
||||
mem_limit: 192m
|
||||
cpus: 0.5
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
s3-store:
|
||||
image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8
|
||||
command: [mini, -dir=/data, -ip=s3-store, -s3=false, -admin.ui=false, -webdav=false, -master.telemetry=false, -master.volumeSizeLimitMB=128, -volume.max=16, -volume.fileSizeLimitMB=11, -volume.concurrentUploadLimitMB=64, -volume.concurrentDownloadLimitMB=64]
|
||||
networks: [storage]
|
||||
volumes: [s3_data:/data]
|
||||
mem_limit: 512m
|
||||
cpus: 1
|
||||
stop_grace_period: 20s
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8888/]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 45
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8
|
||||
command: [s3, -filer=s3-store:8888, -port=8333, -config=/run/secrets/s3_config, -iam=false, -port.iceberg=0, -port.lance=0, -concurrentFileUploadLimit=16, -concurrentUploadLimitMB=32, -readerCacheSizeMB=16]
|
||||
secrets: [s3_config]
|
||||
networks: [backend, storage]
|
||||
mem_limit: 512m
|
||||
environment:
|
||||
GOMEMLIMIT: 384MiB
|
||||
cpus: 0.5
|
||||
depends_on:
|
||||
s3-store:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'nc -z 127.0.0.1 8333']
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
lb:
|
||||
image: haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949
|
||||
ports: ["127.0.0.1:${APP_PORT:-18093}:8080"]
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
networks: [edge, backend]
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
mem_limit: 64m
|
||||
cpus: 0.5
|
||||
stop_grace_period: 12s
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 10
|
||||
postgres:
|
||||
image: postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql/data]
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: [CMD, pg_isready, -h, 127.0.0.1, -U, messenger, -d, messenger]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: ${POSTGRES_CPUS:-1}
|
||||
redis:
|
||||
image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47
|
||||
command: [redis-server, --maxmemory, 32mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"]
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
mem_limit: 64m
|
||||
cpus: 0.25
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af
|
||||
command: [--config=/etc/otel/config.yaml]
|
||||
volumes: [./infra/otel-collector.yaml:/etc/otel/config.yaml:ro]
|
||||
networks: [backend, edge]
|
||||
ports: ["127.0.0.1:${COLLECTOR_PORT:-18896}:8888"]
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
environment:
|
||||
GOMEMLIMIT: 192MiB
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996
|
||||
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.path=/prometheus, --storage.tsdb.retention.time=24h, --storage.tsdb.retention.size=256MB]
|
||||
volumes: [./infra/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro, ./infra/alerts.yaml:/etc/prometheus/alerts.yaml:ro, metrics_data:/prometheus]
|
||||
networks: [backend, edge]
|
||||
ports: ["127.0.0.1:${PROMETHEUS_PORT:-19096}:9090"]
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
loki:
|
||||
image: grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd
|
||||
command: [-config.file=/etc/loki/config.yaml]
|
||||
volumes: [./infra/loki.yaml:/etc/loki/config.yaml:ro, logs_data:/loki]
|
||||
networks: [backend, edge]
|
||||
ports: ["127.0.0.1:${LOKI_PORT:-13106}:3100"]
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
environment:
|
||||
GOMEMLIMIT: 192MiB
|
||||
tempo:
|
||||
image: grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7
|
||||
command: [-config.file=/etc/tempo/config.yaml]
|
||||
volumes: [./infra/tempo.yaml:/etc/tempo/config.yaml:ro, traces_data:/var/tempo]
|
||||
networks: [backend, edge]
|
||||
ports: ["127.0.0.1:${TEMPO_PORT:-13206}:3200"]
|
||||
mem_limit: 512m
|
||||
cpus: 0.5
|
||||
environment:
|
||||
GOMEMLIMIT: 384MiB
|
||||
grafana:
|
||||
image: grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d
|
||||
volumes: [./infra/grafana/provisioning:/etc/grafana/provisioning:ro, ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro, grafana_data:/var/lib/grafana]
|
||||
networks: [backend, edge]
|
||||
ports: ["127.0.0.1:${GRAFANA_PORT:-13006}:3000"]
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
||||
GF_AUTH_DISABLE_LOGIN_FORM: "true"
|
||||
GF_AUTH_BASIC_ENABLED: "false"
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_USERS_VIEWERS_CAN_EDIT: "true"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
|
||||
volumes:
|
||||
metrics_data:
|
||||
logs_data:
|
||||
traces_data:
|
||||
grafana_data:
|
||||
postgres_data:
|
||||
broker_data:
|
||||
s3_data:
|
||||
networks:
|
||||
edge:
|
||||
storage:
|
||||
internal: true
|
||||
backend:
|
||||
internal: true
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_admin_key:
|
||||
file: ./secrets/s3_admin_key
|
||||
s3_admin_secret:
|
||||
file: ./secrets/s3_admin_secret
|
||||
s3_api_key:
|
||||
file: ./secrets/s3_api_key
|
||||
s3_api_secret:
|
||||
file: ./secrets/s3_api_secret
|
||||
s3_config:
|
||||
file: ./secrets/s3_config
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
s3_worker_key:
|
||||
file: ./secrets/s3_worker_key
|
||||
s3_worker_secret:
|
||||
file: ./secrets/s3_worker_secret
|
||||
@@ -0,0 +1,100 @@
|
||||
{
|
||||
"duration_seconds": 56.09580111503601,
|
||||
"confirmed_messages": 135,
|
||||
"confirmed_attachments_ready": 5,
|
||||
"all_confirmed_messages_retained": true,
|
||||
"collector_samples": [
|
||||
{
|
||||
"elapsed": 2.1163909435272217,
|
||||
"memory": "58.21MiB / 256MiB",
|
||||
"cpu": "1.00%"
|
||||
},
|
||||
{
|
||||
"elapsed": 14.031727075576782,
|
||||
"memory": "58.21MiB / 256MiB",
|
||||
"cpu": "0.59%"
|
||||
},
|
||||
{
|
||||
"elapsed": 25.931473970413208,
|
||||
"memory": "58.77MiB / 256MiB",
|
||||
"cpu": "0.62%"
|
||||
},
|
||||
{
|
||||
"elapsed": 37.78569412231445,
|
||||
"memory": "62.01MiB / 256MiB",
|
||||
"cpu": "0.69%"
|
||||
},
|
||||
{
|
||||
"elapsed": 49.66645312309265,
|
||||
"memory": "62.05MiB / 256MiB",
|
||||
"cpu": "1.13%"
|
||||
}
|
||||
],
|
||||
"before": {
|
||||
"otelcol_exporter_queue_capacity": 256.0,
|
||||
"otelcol_exporter_queue_size": 0.0,
|
||||
"otelcol_exporter_send_failed_log_records_total": 0.0,
|
||||
"otelcol_exporter_send_failed_metric_points_total": 0.0,
|
||||
"otelcol_exporter_send_failed_spans_total": 0.0,
|
||||
"otelcol_exporter_sent_log_records_total": 824.0,
|
||||
"otelcol_exporter_sent_metric_points_total": 16842.0,
|
||||
"otelcol_exporter_sent_spans_total": 1783.0,
|
||||
"otelcol_process_cpu_seconds_total": 2.12,
|
||||
"otelcol_process_memory_rss_bytes": 203309056.0,
|
||||
"otelcol_process_runtime_heap_alloc_bytes": 32249488.0,
|
||||
"otelcol_process_runtime_total_alloc_bytes_total": 333861280.0,
|
||||
"otelcol_process_runtime_total_sys_memory_bytes": 62215432.0,
|
||||
"otelcol_process_uptime_seconds_total": 401.657121304,
|
||||
"otelcol_processor_accepted_log_records_total": 824.0,
|
||||
"otelcol_processor_accepted_metric_points_total": 16917.0,
|
||||
"otelcol_processor_accepted_spans_total": 1783.0,
|
||||
"otelcol_processor_batch_batch_send_size_bucket": 10026.0,
|
||||
"otelcol_processor_batch_batch_send_size_sum": 19449.0,
|
||||
"otelcol_processor_batch_batch_send_size_count": 478.0,
|
||||
"otelcol_processor_batch_batch_size_trigger_send_total": 11.0,
|
||||
"otelcol_processor_batch_metadata_cardinality": 3.0,
|
||||
"otelcol_processor_batch_timeout_trigger_send_total": 467.0,
|
||||
"otelcol_processor_incoming_items_total": 19524.0,
|
||||
"otelcol_processor_outgoing_items_total": 19524.0,
|
||||
"otelcol_receiver_accepted_log_records_total": 824.0,
|
||||
"otelcol_receiver_accepted_metric_points_total": 16917.0,
|
||||
"otelcol_receiver_accepted_spans_total": 1783.0,
|
||||
"otelcol_receiver_refused_log_records_total": 0.0,
|
||||
"otelcol_receiver_refused_metric_points_total": 0.0,
|
||||
"otelcol_receiver_refused_spans_total": 0.0
|
||||
},
|
||||
"during": {
|
||||
"otelcol_exporter_queue_capacity": 256.0,
|
||||
"otelcol_exporter_queue_size": 37.0,
|
||||
"otelcol_exporter_send_failed_log_records_total": 0.0,
|
||||
"otelcol_exporter_send_failed_metric_points_total": 0.0,
|
||||
"otelcol_exporter_send_failed_spans_total": 156.0,
|
||||
"otelcol_exporter_sent_log_records_total": 1021.0,
|
||||
"otelcol_exporter_sent_metric_points_total": 19388.0,
|
||||
"otelcol_exporter_sent_spans_total": 1801.0,
|
||||
"otelcol_process_cpu_seconds_total": 2.54,
|
||||
"otelcol_process_memory_rss_bytes": 206245888.0,
|
||||
"otelcol_process_runtime_heap_alloc_bytes": 41796440.0,
|
||||
"otelcol_process_runtime_total_alloc_bytes_total": 385908760.0,
|
||||
"otelcol_process_runtime_total_sys_memory_bytes": 66409736.0,
|
||||
"otelcol_process_uptime_seconds_total": 456.688764525,
|
||||
"otelcol_processor_accepted_log_records_total": 1021.0,
|
||||
"otelcol_processor_accepted_metric_points_total": 19388.0,
|
||||
"otelcol_processor_accepted_spans_total": 2356.0,
|
||||
"otelcol_processor_batch_batch_send_size_bucket": 12845.0,
|
||||
"otelcol_processor_batch_batch_send_size_sum": 22765.0,
|
||||
"otelcol_processor_batch_batch_send_size_count": 606.0,
|
||||
"otelcol_processor_batch_batch_size_trigger_send_total": 11.0,
|
||||
"otelcol_processor_batch_metadata_cardinality": 3.0,
|
||||
"otelcol_processor_batch_timeout_trigger_send_total": 595.0,
|
||||
"otelcol_processor_incoming_items_total": 22765.0,
|
||||
"otelcol_processor_outgoing_items_total": 22765.0,
|
||||
"otelcol_receiver_accepted_log_records_total": 1021.0,
|
||||
"otelcol_receiver_accepted_metric_points_total": 19388.0,
|
||||
"otelcol_receiver_accepted_spans_total": 2356.0,
|
||||
"otelcol_receiver_refused_log_records_total": 0.0,
|
||||
"otelcol_receiver_refused_metric_points_total": 0.0,
|
||||
"otelcol_receiver_refused_spans_total": 0.0
|
||||
},
|
||||
"recovered_trace_id": "1555228135a04659bac5654623c7a552"
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
{
|
||||
"mode": "balanced25",
|
||||
"enabled": 1,
|
||||
"sample_ratio": 0.25,
|
||||
"repeat": 0,
|
||||
"arrival_rps": 30,
|
||||
"scheduled_requests": 600,
|
||||
"confirmed_requests": 600,
|
||||
"load_wall_seconds": 19.970493208,
|
||||
"successful_rps": 30.044325583288316,
|
||||
"p50_seconds": 0.003056625000001034,
|
||||
"p95_seconds": 0.010586957999999314,
|
||||
"p99_seconds": 0.017221916999998754,
|
||||
"errors": [],
|
||||
"image_jobs_ready": 30,
|
||||
"resource": {
|
||||
"api": {
|
||||
"cpu_seconds": 2.6645160000000008,
|
||||
"peak_sampled_anonymous_bytes": 197382144,
|
||||
"peak_sampled_working_set_bytes": 206495744,
|
||||
"rx_bytes": 4570530
|
||||
},
|
||||
"broker": {
|
||||
"cpu_seconds": 0.15844099999999983,
|
||||
"peak_sampled_anonymous_bytes": 3289088,
|
||||
"peak_sampled_working_set_bytes": 3899392,
|
||||
"rx_bytes": 46110
|
||||
},
|
||||
"dispatcher": {
|
||||
"cpu_seconds": 0.7753760000000005,
|
||||
"peak_sampled_anonymous_bytes": 45830144,
|
||||
"peak_sampled_working_set_bytes": 48340992,
|
||||
"rx_bytes": 472693
|
||||
},
|
||||
"grafana": {
|
||||
"cpu_seconds": 0.34897599999999995,
|
||||
"peak_sampled_anonymous_bytes": 106962944,
|
||||
"peak_sampled_working_set_bytes": 114311168,
|
||||
"rx_bytes": 77173
|
||||
},
|
||||
"lb": {
|
||||
"cpu_seconds": 0.24454800000000043,
|
||||
"peak_sampled_anonymous_bytes": 6942720,
|
||||
"peak_sampled_working_set_bytes": 7999488,
|
||||
"rx_bytes": 4287798
|
||||
},
|
||||
"loki": {
|
||||
"cpu_seconds": 0.16779699999999975,
|
||||
"peak_sampled_anonymous_bytes": 108310528,
|
||||
"peak_sampled_working_set_bytes": 112246784,
|
||||
"rx_bytes": 111200
|
||||
},
|
||||
"otel-collector": {
|
||||
"cpu_seconds": 0.19291599999999987,
|
||||
"peak_sampled_anonymous_bytes": 57331712,
|
||||
"peak_sampled_working_set_bytes": 63811584,
|
||||
"rx_bytes": 863090
|
||||
},
|
||||
"postgres": {
|
||||
"cpu_seconds": 1.631062,
|
||||
"peak_sampled_anonymous_bytes": 15544320,
|
||||
"peak_sampled_working_set_bytes": 38952960,
|
||||
"rx_bytes": 1139116
|
||||
},
|
||||
"prometheus": {
|
||||
"cpu_seconds": 0.1952210000000003,
|
||||
"peak_sampled_anonymous_bytes": 56045568,
|
||||
"peak_sampled_working_set_bytes": 59535360,
|
||||
"rx_bytes": 116869
|
||||
},
|
||||
"redis": {
|
||||
"cpu_seconds": 0.17422799999999983,
|
||||
"peak_sampled_anonymous_bytes": 3067904,
|
||||
"peak_sampled_working_set_bytes": 3612672,
|
||||
"rx_bytes": 119742
|
||||
},
|
||||
"s3": {
|
||||
"cpu_seconds": 0.4350659999999995,
|
||||
"peak_sampled_anonymous_bytes": 69742592,
|
||||
"peak_sampled_working_set_bytes": 71454720,
|
||||
"rx_bytes": 11144479
|
||||
},
|
||||
"s3-store": {
|
||||
"cpu_seconds": 0.9894250000000007,
|
||||
"peak_sampled_anonymous_bytes": 134537216,
|
||||
"peak_sampled_working_set_bytes": 137109504,
|
||||
"rx_bytes": 7486398
|
||||
},
|
||||
"tempo": {
|
||||
"cpu_seconds": 0.11411999999999978,
|
||||
"peak_sampled_anonymous_bytes": 360697856,
|
||||
"peak_sampled_working_set_bytes": 364331008,
|
||||
"rx_bytes": 37382
|
||||
},
|
||||
"worker": {
|
||||
"cpu_seconds": 5.430313999999999,
|
||||
"peak_sampled_anonymous_bytes": 77266944,
|
||||
"peak_sampled_working_set_bytes": 82108416,
|
||||
"rx_bytes": 3761105
|
||||
}
|
||||
},
|
||||
"collector_ingest": {
|
||||
"otelcol_receiver_accepted_spans_total": 456.0,
|
||||
"otelcol_receiver_accepted_log_records_total": 840.0,
|
||||
"otelcol_receiver_accepted_metric_points_total": 1154.0,
|
||||
"otelcol_exporter_sent_spans_total": 456.0,
|
||||
"otelcol_exporter_send_failed_spans_total": 0
|
||||
},
|
||||
"sampled_spans_finished": 456.0,
|
||||
"sampled_spans_not_accepted_after_drain": 0,
|
||||
"rare_probes": [
|
||||
{
|
||||
"request_id": "0c11495cc94443caadcd12bd17a86dd0",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4905514579999988,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "aeacd5bdeb7d0f073b2a2cc44487c22c"
|
||||
},
|
||||
{
|
||||
"request_id": "b0871c2ae52245ce882338cf01bd6a4d",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4968927090000008,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "4d8c492fae68617ec5dbd2bc849de2e9"
|
||||
},
|
||||
{
|
||||
"request_id": "d1d60c7ee15f498b988e031e15dc7d27",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4817318749999941,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "931e742a91866246ac3b77dd062517c6"
|
||||
},
|
||||
{
|
||||
"request_id": "95e79f3e1894491c9355c15e0e79a695",
|
||||
"status": 200,
|
||||
"latency_seconds": 1.1869852499999993,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "34e1ad7e9c366b666caf53045be238e9"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"source_commit": "0e78b2b27de93464a6bd0974026bee4bfe4f602a",
|
||||
"project": "mtusi-lab6-clean-6df218fa",
|
||||
"new_secrets": true,
|
||||
"new_empty_volumes": true,
|
||||
"build_smoke_18_api_tests": true,
|
||||
"real_observability_proof": true,
|
||||
"business_and_telemetry_persistence": true,
|
||||
"temporary_project_removed": true,
|
||||
"seconds": 83.54
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
RUN python3 scripts/bootstrap.py
|
||||
Local configuration ready; no secrets printed.
|
||||
RUN docker compose build api
|
||||
Image mtusi-messenger:lab6 Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 707B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 739B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#3 DONE 1.0s
|
||||
|
||||
#4 [internal] load .dockerignore
|
||||
#4 transferring context: 203B done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
#5 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [internal] load build context
|
||||
#6 transferring context: 88.54kB done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
#7 CACHED
|
||||
|
||||
#8 [5/7] RUN pip install --no-cache-dir -r requirements.txt
|
||||
#8 CACHED
|
||||
|
||||
#9 [6/7] COPY src/ ./src/
|
||||
#9 CACHED
|
||||
|
||||
#10 [2/7] WORKDIR /app
|
||||
#10 CACHED
|
||||
|
||||
#11 [4/7] COPY requirements.txt ./
|
||||
#11 CACHED
|
||||
|
||||
#12 [7/7] COPY migrations/ ./migrations/
|
||||
#12 CACHED
|
||||
|
||||
#13 exporting to image
|
||||
#13 exporting layers done
|
||||
#13 exporting manifest sha256:e533c95082fcc2d03df488d24c04c74c3205f96ebce7d39fcf544e15ed2ac2c9 done
|
||||
#13 exporting config sha256:d5159a690cc9eed35b1e1df5510f9f8595fbb5e303c09bde83ea92197bbfab8c done
|
||||
#13 exporting attestation manifest sha256:91fc899a62ccb18479e1135cbfe8c85ae865353f7315c7c8897a6418fe283c11 done
|
||||
#13 exporting manifest list sha256:a0c792a12285d3f4914e1ce37a8832777d639abd2653a71ff0b581b28f2f5e6b done
|
||||
#13 naming to docker.io/library/mtusi-messenger:lab6 done
|
||||
#13 unpacking to docker.io/library/mtusi-messenger:lab6 done
|
||||
#13 DONE 0.0s
|
||||
|
||||
#14 resolving provenance for metadata file
|
||||
#14 DONE 0.0s
|
||||
Image mtusi-messenger:lab6 Built
|
||||
RUN docker compose up -d --wait --wait-timeout 180
|
||||
Network mtusi-lab6-clean-6df218fa_edge Creating
|
||||
Network mtusi-lab6-clean-6df218fa_edge Created
|
||||
Network mtusi-lab6-clean-6df218fa_backend Creating
|
||||
Network mtusi-lab6-clean-6df218fa_backend Created
|
||||
Network mtusi-lab6-clean-6df218fa_storage Creating
|
||||
Network mtusi-lab6-clean-6df218fa_storage Created
|
||||
Volume mtusi-lab6-clean-6df218fa_logs_data Creating
|
||||
Volume mtusi-lab6-clean-6df218fa_logs_data Created
|
||||
Volume mtusi-lab6-clean-6df218fa_metrics_data Creating
|
||||
Volume mtusi-lab6-clean-6df218fa_metrics_data Created
|
||||
Volume mtusi-lab6-clean-6df218fa_traces_data Creating
|
||||
Volume mtusi-lab6-clean-6df218fa_traces_data Created
|
||||
Volume mtusi-lab6-clean-6df218fa_postgres_data Creating
|
||||
Volume mtusi-lab6-clean-6df218fa_postgres_data Created
|
||||
Volume mtusi-lab6-clean-6df218fa_s3_data Creating
|
||||
Volume mtusi-lab6-clean-6df218fa_s3_data Created
|
||||
Volume mtusi-lab6-clean-6df218fa_broker_data Creating
|
||||
Volume mtusi-lab6-clean-6df218fa_broker_data Created
|
||||
Volume mtusi-lab6-clean-6df218fa_grafana_data Creating
|
||||
Volume mtusi-lab6-clean-6df218fa_grafana_data Created
|
||||
Container mtusi-lab6-clean-6df218fa-grafana-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-prometheus-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-loki-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-tempo-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-grafana-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-prometheus-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-loki-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-tempo-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Created
|
||||
Container mtusi-lab6-clean-6df218fa-lb-1 Creating
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-lb-1 Created
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-tempo-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-prometheus-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-loki-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-grafana-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-prometheus-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-loki-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-tempo-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-grafana-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
|
||||
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Started
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-lb-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-lb-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-prometheus-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-lb-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-loki-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-tempo-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-grafana-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Waiting
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-loki-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Exited
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Exited
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-tempo-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-grafana-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-prometheus-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Healthy
|
||||
Container mtusi-lab6-clean-6df218fa-lb-1 Healthy
|
||||
RUN python3 scripts/wait_ready.py
|
||||
Ready through load balancer.
|
||||
RUN make check test test-student BASE_URL=http://127.0.0.1:18094
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:18094"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
BASE_URL="http://127.0.0.1:18094" python3 -m unittest discover -s student-tests -p 'test_*.py' -v
|
||||
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
|
||||
test_cursor_scope_and_validation (test_api.APIContract) ... ok
|
||||
test_foreign_chat (test_api.APIContract) ... ok
|
||||
test_idempotency_concurrent_replicas (test_api.APIContract) ... ok
|
||||
test_idempotency_conflicting_race (test_api.APIContract) ... ok
|
||||
test_idempotency_invalid_keys (test_api.APIContract) ... ok
|
||||
test_idempotency_scope (test_api.APIContract) ... ok
|
||||
test_idempotency_semantic_body (test_api.APIContract) ... ok
|
||||
test_invalid_chat_creation (test_api.APIContract) ... ok
|
||||
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
|
||||
test_protocol_error_headers (test_api.APIContract) ... ok
|
||||
test_registration_race (test_api.APIContract) ... ok
|
||||
test_unicode_limits (test_api.APIContract) ... ok
|
||||
test_unicode_nul_round_trip (test_api.APIContract) ... ok
|
||||
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
|
||||
test_async_variants_and_replay (test_images.Images) ... ok
|
||||
test_mime_bytes_pixels_and_filename (test_images.Images) ... ok
|
||||
test_private_and_cross_chat (test_images.Images) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 18 tests in 4.950s
|
||||
|
||||
OK
|
||||
RUN python3 -u student-tests/observability.py proof
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Paused
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Starting
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Started
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Unpaused
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Running
|
||||
PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.
|
||||
RUN python3 -u student-tests/persistence_telemetry.py
|
||||
PASS: full business restart, original Bearer, historical Tempo trace, Loki logs and fixed-time Prometheus sample.
|
||||
RUN docker compose down -v --remove-orphans
|
||||
Container mtusi-lab6-clean-6df218fa-loki-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-lb-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-grafana-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-tempo-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-prometheus-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-dispatcher-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-lb-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-lb-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-lb-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-worker-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-grafana-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-grafana-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-prometheus-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-prometheus-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-grafana-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-prometheus-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-broker-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-api-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-api-2 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-migrate-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-bucket-bootstrap-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-redis-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-postgres-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-loki-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-loki-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-loki-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-s3-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Stopping
|
||||
Container mtusi-lab6-clean-6df218fa-otel-collector-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-s3-store-1 Removed
|
||||
Container mtusi-lab6-clean-6df218fa-tempo-1 Stopped
|
||||
Container mtusi-lab6-clean-6df218fa-tempo-1 Removing
|
||||
Container mtusi-lab6-clean-6df218fa-tempo-1 Removed
|
||||
Volume mtusi-lab6-clean-6df218fa_broker_data Removing
|
||||
Network mtusi-lab6-clean-6df218fa_storage Removing
|
||||
Volume mtusi-lab6-clean-6df218fa_metrics_data Removing
|
||||
Volume mtusi-lab6-clean-6df218fa_grafana_data Removing
|
||||
Volume mtusi-lab6-clean-6df218fa_traces_data Removing
|
||||
Network mtusi-lab6-clean-6df218fa_edge Removing
|
||||
Volume mtusi-lab6-clean-6df218fa_s3_data Removing
|
||||
Volume mtusi-lab6-clean-6df218fa_logs_data Removing
|
||||
Volume mtusi-lab6-clean-6df218fa_postgres_data Removing
|
||||
Network mtusi-lab6-clean-6df218fa_backend Removing
|
||||
Volume mtusi-lab6-clean-6df218fa_broker_data Removed
|
||||
Volume mtusi-lab6-clean-6df218fa_metrics_data Removed
|
||||
Volume mtusi-lab6-clean-6df218fa_logs_data Removed
|
||||
Volume mtusi-lab6-clean-6df218fa_grafana_data Removed
|
||||
Volume mtusi-lab6-clean-6df218fa_traces_data Removed
|
||||
Volume mtusi-lab6-clean-6df218fa_s3_data Removed
|
||||
Volume mtusi-lab6-clean-6df218fa_postgres_data Removed
|
||||
Network mtusi-lab6-clean-6df218fa_storage Removed
|
||||
Network mtusi-lab6-clean-6df218fa_backend Removed
|
||||
Network mtusi-lab6-clean-6df218fa_edge Removed
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 79 KiB |
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"off": {
|
||||
"runs": 3,
|
||||
"requests": 1800,
|
||||
"ready_images": 90,
|
||||
"p95_ms": 9.81791700002077,
|
||||
"p99_ms": 15.300541999977213,
|
||||
"api_cpu_seconds": 2.3706709999999998,
|
||||
"worker_cpu_seconds": 5.463901,
|
||||
"collector_cpu_seconds": 0.053601000000000454,
|
||||
"api_sum_sampled_anon_peak_mib": 159.59765625,
|
||||
"collector_anon_peak_mib": 56.015625,
|
||||
"accepted_spans": 0,
|
||||
"accepted_logs": 0,
|
||||
"accepted_metric_points": 0,
|
||||
"collector_receive_kib": 13.2080078125,
|
||||
"sampled_spans_not_accepted": 0,
|
||||
"error_probe_traces_kept": 0,
|
||||
"error_probes": 1,
|
||||
"slow_probe_traces_kept": 0,
|
||||
"slow_probes": 11
|
||||
},
|
||||
"full": {
|
||||
"runs": 3,
|
||||
"requests": 1800,
|
||||
"ready_images": 90,
|
||||
"p95_ms": 9.695250000049782,
|
||||
"p99_ms": 14.552875000049426,
|
||||
"api_cpu_seconds": 2.7994320000000004,
|
||||
"worker_cpu_seconds": 5.533063,
|
||||
"collector_cpu_seconds": 0.19410300000000014,
|
||||
"api_sum_sampled_anon_peak_mib": 147.90234375,
|
||||
"collector_anon_peak_mib": 56.02734375,
|
||||
"accepted_spans": 1950.0,
|
||||
"accepted_logs": 840.0,
|
||||
"accepted_metric_points": 1011.0,
|
||||
"collector_receive_kib": 1067.025390625,
|
||||
"sampled_spans_not_accepted": 0,
|
||||
"error_probe_traces_kept": 4,
|
||||
"error_probes": 4,
|
||||
"slow_probe_traces_kept": 8,
|
||||
"slow_probes": 8
|
||||
},
|
||||
"sample10": {
|
||||
"runs": 3,
|
||||
"requests": 1800,
|
||||
"ready_images": 90,
|
||||
"p95_ms": 9.149458999957005,
|
||||
"p99_ms": 13.731374999991885,
|
||||
"api_cpu_seconds": 2.540737,
|
||||
"worker_cpu_seconds": 5.2699810000000005,
|
||||
"collector_cpu_seconds": 0.16662400000000055,
|
||||
"api_sum_sampled_anon_peak_mib": 168.65625,
|
||||
"collector_anon_peak_mib": 58.78125,
|
||||
"accepted_spans": 176.0,
|
||||
"accepted_logs": 840.0,
|
||||
"accepted_metric_points": 987.0,
|
||||
"collector_receive_kib": 720.869140625,
|
||||
"sampled_spans_not_accepted": 0,
|
||||
"error_probe_traces_kept": 0,
|
||||
"error_probes": 1,
|
||||
"slow_probe_traces_kept": 0,
|
||||
"slow_probes": 11
|
||||
},
|
||||
"sample25": {
|
||||
"runs": 3,
|
||||
"requests": 1800,
|
||||
"ready_images": 90,
|
||||
"p95_ms": 10.556667000003017,
|
||||
"p99_ms": 15.699083000015435,
|
||||
"api_cpu_seconds": 2.6178429999999997,
|
||||
"worker_cpu_seconds": 5.2202720000000005,
|
||||
"collector_cpu_seconds": 0.18281499999999973,
|
||||
"api_sum_sampled_anon_peak_mib": 147.81640625,
|
||||
"collector_anon_peak_mib": 58.71875,
|
||||
"accepted_spans": 467.0,
|
||||
"accepted_logs": 840.0,
|
||||
"accepted_metric_points": 1011.0,
|
||||
"collector_receive_kib": 802.8349609375,
|
||||
"sampled_spans_not_accepted": 0,
|
||||
"error_probe_traces_kept": 1,
|
||||
"error_probes": 2,
|
||||
"slow_probe_traces_kept": 3,
|
||||
"slow_probes": 10
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18093"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 1, "first_recovery_seconds": 2.602}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,28 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:18093"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
BASE_URL="http://127.0.0.1:18093" python3 -m unittest discover -s student-tests -p 'test_*.py' -v
|
||||
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
|
||||
test_cursor_scope_and_validation (test_api.APIContract) ... ok
|
||||
test_foreign_chat (test_api.APIContract) ... ok
|
||||
test_idempotency_concurrent_replicas (test_api.APIContract) ... ok
|
||||
test_idempotency_conflicting_race (test_api.APIContract) ... ok
|
||||
test_idempotency_invalid_keys (test_api.APIContract) ... ok
|
||||
test_idempotency_scope (test_api.APIContract) ... ok
|
||||
test_idempotency_semantic_body (test_api.APIContract) ... ok
|
||||
test_invalid_chat_creation (test_api.APIContract) ... ok
|
||||
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
|
||||
test_protocol_error_headers (test_api.APIContract) ... ok
|
||||
test_registration_race (test_api.APIContract) ... ok
|
||||
test_unicode_limits (test_api.APIContract) ... ok
|
||||
test_unicode_nul_round_trip (test_api.APIContract) ... ok
|
||||
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
|
||||
test_async_variants_and_replay (test_images.Images) ... ok
|
||||
test_mime_bytes_pixels_and_filename (test_images.Images) ... ok
|
||||
test_private_and_cross_chat (test_images.Images) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 18 tests in 5.414s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1 @@
|
||||
PASS: Python syntax; original README/COURSE/CONTRIBUTING/contracts/public tests unchanged; actual local PostgreSQL password and S3 credentials absent from tracked/new files. Non-secret internal Redis service URLs intentionally appear in bootstrap.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 54 KiB |
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 165 KiB |
File diff suppressed because it is too large
Load Diff
|
After Width: | Height: | Size: 145 KiB |
@@ -0,0 +1,8 @@
|
||||
Container mtusi-lab6-worker-1 Stopping
|
||||
Container mtusi-lab6-worker-1 Stopped
|
||||
Container mtusi-lab6-s3-1 Paused
|
||||
Container mtusi-lab6-worker-1 Starting
|
||||
Container mtusi-lab6-worker-1 Started
|
||||
Container mtusi-lab6-s3-1 Unpaused
|
||||
Container mtusi-lab6-worker-1 Running
|
||||
PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18093"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,119 @@
|
||||
{
|
||||
"invariants": {
|
||||
"commit_before_publish_recovered": true,
|
||||
"publish_before_marker_duplicate_safe": true,
|
||||
"crashes_before_after_side_effect_and_commit": true,
|
||||
"invalid_envelope_observable_dlq": true,
|
||||
"duplicate_and_reorder_no_second_effect": true,
|
||||
"concurrent_duplicate_single_claim": true,
|
||||
"stale_running_version_fenced": true,
|
||||
"old_delivery_after_new_ready_no_overwrite": true,
|
||||
"failed_registry_explicit_replay_and_ready_refusal": true,
|
||||
"transient_backoff_exhaustion_then_replay": true,
|
||||
"invalid_decode_permanent_failed": true,
|
||||
"cleanup_preserves_current_and_fresh_failed": true,
|
||||
"ready_staging_cleanup_idempotent": true
|
||||
},
|
||||
"faults": [
|
||||
{
|
||||
"point": "before_side_effect",
|
||||
"exit_code": 77,
|
||||
"state_at_crash": "processing",
|
||||
"recovered": "ready",
|
||||
"attempts": 2,
|
||||
"result_objects": 4
|
||||
},
|
||||
{
|
||||
"point": "after_original",
|
||||
"exit_code": 77,
|
||||
"state_at_crash": "processing",
|
||||
"recovered": "ready",
|
||||
"attempts": 2,
|
||||
"result_objects": 4
|
||||
},
|
||||
{
|
||||
"point": "after_objects",
|
||||
"exit_code": 77,
|
||||
"state_at_crash": "processing",
|
||||
"recovered": "ready",
|
||||
"attempts": 2,
|
||||
"result_objects": 4
|
||||
},
|
||||
{
|
||||
"point": "after_commit",
|
||||
"exit_code": 77,
|
||||
"state_at_crash": "ready",
|
||||
"recovered": "ready",
|
||||
"attempts": 1,
|
||||
"result_objects": 4
|
||||
}
|
||||
],
|
||||
"retry": {
|
||||
"transitions": [
|
||||
{
|
||||
"state": "queued",
|
||||
"attempt": 1,
|
||||
"error_code": "storage_unavailable"
|
||||
},
|
||||
{
|
||||
"state": "failed",
|
||||
"attempt": 2,
|
||||
"error_code": "retry_exhausted"
|
||||
}
|
||||
],
|
||||
"seconds": 1.983,
|
||||
"replayed_attempt": 3
|
||||
},
|
||||
"transforms": [
|
||||
{
|
||||
"kind": "alpha",
|
||||
"size": [
|
||||
20,
|
||||
10
|
||||
],
|
||||
"exif_removed": true
|
||||
},
|
||||
{
|
||||
"kind": "exif",
|
||||
"size": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"exif_removed": true
|
||||
},
|
||||
{
|
||||
"kind": "small",
|
||||
"size": [
|
||||
5,
|
||||
3
|
||||
],
|
||||
"exif_removed": true
|
||||
}
|
||||
],
|
||||
"cleanup": [
|
||||
{
|
||||
"apply": false,
|
||||
"retention_seconds": 86400,
|
||||
"counts": {
|
||||
"ready_staging": 397,
|
||||
"expired_staging": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"apply": true,
|
||||
"retention_seconds": 86400,
|
||||
"counts": {
|
||||
"ready_staging": 397,
|
||||
"expired_staging": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"apply": true,
|
||||
"retention_seconds": 86400,
|
||||
"counts": {
|
||||
"ready_staging": 0,
|
||||
"expired_staging": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,546 @@
|
||||
[
|
||||
{
|
||||
"container": "cf2cfedf29de75b2458ec532a42f264cdb7ef71435900a7ef1c478d9d6b2e61f",
|
||||
"name": "/mtusi-lab6-api-1",
|
||||
"service": "api",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:44.122480904Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_secret",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/database_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/cache_url",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b",
|
||||
"name": "/mtusi-lab6-api-2",
|
||||
"service": "api",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:44.196103962Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/database_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/cache_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_secret",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "13d6ed83fc36b28aa57fc7cf302d9624098934b1f9c6ec68254a2b1a5f95228d",
|
||||
"name": "/mtusi-lab6-broker-1",
|
||||
"service": "broker",
|
||||
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:30.515512908Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 201326592,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"6379/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "mtusi-lab6_broker_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "076e71ce04ff5adbc19738e63a8ed319ad6f645183e6e65e47aec7dd97afe926",
|
||||
"name": "/mtusi-lab6-bucket-bootstrap-1",
|
||||
"service": "bucket-bootstrap",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": false,
|
||||
"started_at": "2026-11-20T08:45:43.045813539Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_admin_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_admin_secret",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "b2de6e448b4102ff9f25833bf99cd5a9978947dae0c3e28e743c3ef18f58b943",
|
||||
"name": "/mtusi-lab6-dispatcher-1",
|
||||
"service": "dispatcher",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:43.627780758Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/database_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/broker_url",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "54edeaec98aff44bfe34f8c103c33694187550a9b690cde7648017ad323d7310",
|
||||
"name": "/mtusi-lab6-grafana-1",
|
||||
"service": "grafana",
|
||||
"image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d",
|
||||
"user": "472",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:30.734589337Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"3000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "13006"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/grafana/provisioning",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/var/lib/grafana/dashboards",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/lib/grafana",
|
||||
"name": "mtusi-lab6_grafana_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "7db8217af00aa7be6a7e3d19f656faf329598ef85bfc8fb2fc2d15ec0cf72c28",
|
||||
"name": "/mtusi-lab6-lb-1",
|
||||
"service": "lb",
|
||||
"image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
|
||||
"user": "haproxy",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:49.83573792Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "18093"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/usr/local/etc/haproxy/haproxy.cfg",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "712535508b37fe699452fb2a4ed62a005a8cfa7a14e2b9827a9c82cd4ec982c9",
|
||||
"name": "/mtusi-lab6-loki-1",
|
||||
"service": "loki",
|
||||
"image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd",
|
||||
"user": "10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:31.075549983Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"3100/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "13106"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/loki/config.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/loki",
|
||||
"name": "mtusi-lab6_logs_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "6374cc6d3e2b163264c18e1181ea5be5b53f365a5324686a908ea8f94861a6c7",
|
||||
"name": "/mtusi-lab6-migrate-1",
|
||||
"service": "migrate",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": false,
|
||||
"started_at": "2026-11-20T08:45:42.539353205Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/cache_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_secret",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/database_url",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "89012420d895c4c7c0d9fdaf7c1ff40c3a5f97dca279a0ef8cec0be3903dce42",
|
||||
"name": "/mtusi-lab6-otel-collector-1",
|
||||
"service": "otel-collector",
|
||||
"image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:30.652417444Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"4317/tcp": null,
|
||||
"4318/tcp": null,
|
||||
"55679/tcp": null,
|
||||
"8888/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "18896"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/otel/config.yaml",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "2257079aebc7a27eda5607608d7c4e7df3d1a196eb2c65d7236a1b60f7466c91",
|
||||
"name": "/mtusi-lab6-postgres-1",
|
||||
"service": "postgres",
|
||||
"image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:30.708099365Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"5432/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/lib/postgresql/data",
|
||||
"name": "mtusi-lab6_postgres_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/postgres_password",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "937df26581a582233f77674d7c66a0b7fbba5fffd2377977ebd13857cd510a66",
|
||||
"name": "/mtusi-lab6-prometheus-1",
|
||||
"service": "prometheus",
|
||||
"image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996",
|
||||
"user": "nobody",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:30.770516143Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"9090/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "19096"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/prometheus",
|
||||
"name": "mtusi-lab6_metrics_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/prometheus/prometheus.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/prometheus/alerts.yaml",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "cc255a9bdb8967c853de2a4f167fe269a1a4f004e0b88ab174c79397212fd2a4",
|
||||
"name": "/mtusi-lab6-redis-1",
|
||||
"service": "redis",
|
||||
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:30.676898647Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 250000000,
|
||||
"ports": {
|
||||
"6379/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "db7892dbab7e8fc10b9d4431ee816406d604a540d7068b70aef208767168a178"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "c4564618c7c266a67ed1f6a05989f8156ca42582a1860fe8d868fc758d18d4f3",
|
||||
"name": "/mtusi-lab6-s3-1",
|
||||
"service": "s3",
|
||||
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:30.688051331Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"18080/tcp": null,
|
||||
"18888/tcp": null,
|
||||
"19333/tcp": null,
|
||||
"7333/tcp": null,
|
||||
"8080/tcp": null,
|
||||
"8333/tcp": null,
|
||||
"8888/tcp": null,
|
||||
"9333/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_config",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "3f039af87109799a85e80069eb418dd6c13b9b036ffc46fb0e6ed1bfc21407de"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "d3abacb0bdaf348f062e5ec77599ea8a0450872a17b450b190ba7bce78166129",
|
||||
"name": "/mtusi-lab6-s3-store-1",
|
||||
"service": "s3-store",
|
||||
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:31.797812564Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"18080/tcp": null,
|
||||
"18888/tcp": null,
|
||||
"19333/tcp": null,
|
||||
"7333/tcp": null,
|
||||
"8080/tcp": null,
|
||||
"8333/tcp": null,
|
||||
"8888/tcp": null,
|
||||
"9333/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "mtusi-lab6_s3_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "9cfc7a5053725737ecfc90211eed31a0c97a15aeb63ca270b9596c4af69de808",
|
||||
"name": "/mtusi-lab6-tempo-1",
|
||||
"service": "tempo",
|
||||
"image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:40.551529375Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"3200/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "13206"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/tempo",
|
||||
"name": "mtusi-lab6_traces_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/tempo/config.yaml",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "ef14a69e994f9f18fdb3438a03a28a974a3fcfc564f2d0eedeff3949a8f9bb29",
|
||||
"name": "/mtusi-lab6-worker-1",
|
||||
"service": "worker",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:44.122215389Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/database_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/broker_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_worker_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_worker_secret",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,546 @@
|
||||
[
|
||||
{
|
||||
"container": "cf2cfedf29de75b2458ec532a42f264cdb7ef71435900a7ef1c478d9d6b2e61f",
|
||||
"name": "/mtusi-lab6-api-1",
|
||||
"service": "api",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:43:28.014761078Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_secret",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/database_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/cache_url",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b",
|
||||
"name": "/mtusi-lab6-api-2",
|
||||
"service": "api",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:43:28.105739751Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/database_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/cache_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_secret",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "13d6ed83fc36b28aa57fc7cf302d9624098934b1f9c6ec68254a2b1a5f95228d",
|
||||
"name": "/mtusi-lab6-broker-1",
|
||||
"service": "broker",
|
||||
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:20:26.890287569Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 201326592,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"6379/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "mtusi-lab6_broker_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "076e71ce04ff5adbc19738e63a8ed319ad6f645183e6e65e47aec7dd97afe926",
|
||||
"name": "/mtusi-lab6-bucket-bootstrap-1",
|
||||
"service": "bucket-bootstrap",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": false,
|
||||
"started_at": "2026-11-20T08:40:43.538584225Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_admin_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_admin_secret",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "b2de6e448b4102ff9f25833bf99cd5a9978947dae0c3e28e743c3ef18f58b943",
|
||||
"name": "/mtusi-lab6-dispatcher-1",
|
||||
"service": "dispatcher",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:43:28.015064203Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/database_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/broker_url",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "54edeaec98aff44bfe34f8c103c33694187550a9b690cde7648017ad323d7310",
|
||||
"name": "/mtusi-lab6-grafana-1",
|
||||
"service": "grafana",
|
||||
"image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d",
|
||||
"user": "472",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:21:11.514848908Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"3000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "13006"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/lib/grafana",
|
||||
"name": "mtusi-lab6_grafana_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/grafana/provisioning",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/var/lib/grafana/dashboards",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "7db8217af00aa7be6a7e3d19f656faf329598ef85bfc8fb2fc2d15ec0cf72c28",
|
||||
"name": "/mtusi-lab6-lb-1",
|
||||
"service": "lb",
|
||||
"image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949",
|
||||
"user": "haproxy",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:20:40.887085261Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "18093"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/usr/local/etc/haproxy/haproxy.cfg",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "712535508b37fe699452fb2a4ed62a005a8cfa7a14e2b9827a9c82cd4ec982c9",
|
||||
"name": "/mtusi-lab6-loki-1",
|
||||
"service": "loki",
|
||||
"image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd",
|
||||
"user": "10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:21:11.513864698Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"3100/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "13106"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/loki/config.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/loki",
|
||||
"name": "mtusi-lab6_logs_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "6374cc6d3e2b163264c18e1181ea5be5b53f365a5324686a908ea8f94861a6c7",
|
||||
"name": "/mtusi-lab6-migrate-1",
|
||||
"service": "migrate",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": false,
|
||||
"started_at": "2026-11-20T08:40:43.038416658Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/database_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/cache_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_api_secret",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "89012420d895c4c7c0d9fdaf7c1ff40c3a5f97dca279a0ef8cec0be3903dce42",
|
||||
"name": "/mtusi-lab6-otel-collector-1",
|
||||
"service": "otel-collector",
|
||||
"image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:21:11.514654968Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"4317/tcp": null,
|
||||
"4318/tcp": null,
|
||||
"55679/tcp": null,
|
||||
"8888/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "18896"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/otel/config.yaml",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "2257079aebc7a27eda5607608d7c4e7df3d1a196eb2c65d7236a1b60f7466c91",
|
||||
"name": "/mtusi-lab6-postgres-1",
|
||||
"service": "postgres",
|
||||
"image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:20:26.86209481Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"5432/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/lib/postgresql/data",
|
||||
"name": "mtusi-lab6_postgres_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/postgres_password",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "937df26581a582233f77674d7c66a0b7fbba5fffd2377977ebd13857cd510a66",
|
||||
"name": "/mtusi-lab6-prometheus-1",
|
||||
"service": "prometheus",
|
||||
"image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996",
|
||||
"user": "nobody",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:21:11.515853797Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"9090/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "19096"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/prometheus",
|
||||
"name": "mtusi-lab6_metrics_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/prometheus/prometheus.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/prometheus/alerts.yaml",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "cc255a9bdb8967c853de2a4f167fe269a1a4f004e0b88ab174c79397212fd2a4",
|
||||
"name": "/mtusi-lab6-redis-1",
|
||||
"service": "redis",
|
||||
"image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:20:26.875269419Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 250000000,
|
||||
"ports": {
|
||||
"6379/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "db7892dbab7e8fc10b9d4431ee816406d604a540d7068b70aef208767168a178"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "c4564618c7c266a67ed1f6a05989f8156ca42582a1860fe8d868fc758d18d4f3",
|
||||
"name": "/mtusi-lab6-s3-1",
|
||||
"service": "s3",
|
||||
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:20:31.57071865Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"18080/tcp": null,
|
||||
"18888/tcp": null,
|
||||
"19333/tcp": null,
|
||||
"7333/tcp": null,
|
||||
"8080/tcp": null,
|
||||
"8333/tcp": null,
|
||||
"8888/tcp": null,
|
||||
"9333/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_config",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "3f039af87109799a85e80069eb418dd6c13b9b036ffc46fb0e6ed1bfc21407de"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "d3abacb0bdaf348f062e5ec77599ea8a0450872a17b450b190ba7bce78166129",
|
||||
"name": "/mtusi-lab6-s3-store-1",
|
||||
"service": "s3-store",
|
||||
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:20:26.891493875Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"18080/tcp": null,
|
||||
"18888/tcp": null,
|
||||
"19333/tcp": null,
|
||||
"7333/tcp": null,
|
||||
"8080/tcp": null,
|
||||
"8333/tcp": null,
|
||||
"8888/tcp": null,
|
||||
"9333/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "mtusi-lab6_s3_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "9cfc7a5053725737ecfc90211eed31a0c97a15aeb63ca270b9596c4af69de808",
|
||||
"name": "/mtusi-lab6-tempo-1",
|
||||
"service": "tempo",
|
||||
"image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:28:48.715669396Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"3200/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "13206"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/tempo",
|
||||
"name": "mtusi-lab6_traces_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/tempo/config.yaml",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "ef14a69e994f9f18fdb3438a03a28a974a3fcfc564f2d0eedeff3949a8f9bb29",
|
||||
"name": "/mtusi-lab6-worker-1",
|
||||
"service": "worker",
|
||||
"image": "mtusi-messenger:lab6",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T08:45:08.489572558Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/database_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/broker_url",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_worker_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/secrets/s3_worker_secret",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,989 @@
|
||||
{
|
||||
"trace_id": "dd1cb3e17dc547bc8bd3f94033e39f7a",
|
||||
"retry_observed": true,
|
||||
"trace": {
|
||||
"batches": [
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "067e0d5099614580ab61516259b76c14"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab6.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.36.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-api"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "k4UL1Qk8WSI=",
|
||||
"parentSpanId": "ASNFZ4mrze8=",
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"startTimeUnixNano": "1795164308161361378",
|
||||
"endTimeUnixNano": "1795164308169243229",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "http.request.method",
|
||||
"value": {
|
||||
"stringValue": "POST"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.route",
|
||||
"value": {
|
||||
"stringValue": "/api/v1/chats/{chat_id}/attachments"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.response.status_code",
|
||||
"value": {
|
||||
"intValue": "202"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "wgvjepi5Fj0=",
|
||||
"parentSpanId": "k4UL1Qk8WSI=",
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164308161418698",
|
||||
"endTimeUnixNano": "1795164308162079443",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "6f3FdmFjcfY=",
|
||||
"parentSpanId": "k4UL1Qk8WSI=",
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164308162156009",
|
||||
"endTimeUnixNano": "1795164308162662788",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "rvhnrnpClkQ=",
|
||||
"parentSpanId": "k4UL1Qk8WSI=",
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164308163083649",
|
||||
"endTimeUnixNano": "1795164308164393283",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "qBl9xvUTfq8=",
|
||||
"parentSpanId": "k4UL1Qk8WSI=",
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164308164428226",
|
||||
"endTimeUnixNano": "1795164308168921394",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "R2wwvDNpB68=",
|
||||
"parentSpanId": "qBl9xvUTfq8=",
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164308164676000",
|
||||
"endTimeUnixNano": "1795164308168082844",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "6b019c7baab14c89976d2b4081f1c509"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab6.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.36.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-dispatcher"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "t4R0QxdjlSI=",
|
||||
"parentSpanId": "qBl9xvUTfq8=",
|
||||
"name": "attachment publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"startTimeUnixNano": "1795164308271913285",
|
||||
"endTimeUnixNano": "1795164308274338175",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "messaging.system",
|
||||
"value": {
|
||||
"stringValue": "redis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "messaging.destination.name",
|
||||
"value": {
|
||||
"stringValue": "attachment-jobs"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "nBeLRAx4mXA=",
|
||||
"parentSpanId": "qBl9xvUTfq8=",
|
||||
"name": "attachment publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"startTimeUnixNano": "1795164312922355612",
|
||||
"endTimeUnixNano": "1795164312924775467",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "messaging.system",
|
||||
"value": {
|
||||
"stringValue": "redis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "messaging.destination.name",
|
||||
"value": {
|
||||
"stringValue": "attachment-jobs"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "742be3a8d93342da9117f92711193f78"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab6.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.36.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-worker"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "AqkcRaZWl50=",
|
||||
"parentSpanId": "t4R0QxdjlSI=",
|
||||
"name": "attachment process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"startTimeUnixNano": "1795164309031454501",
|
||||
"endTimeUnixNano": "1795164312029923986",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "messaging.system",
|
||||
"value": {
|
||||
"stringValue": "redis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "messaging.destination.name",
|
||||
"value": {
|
||||
"stringValue": "attachment-jobs"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "job.id",
|
||||
"value": {
|
||||
"stringValue": "9d711cdf-347d-4590-91da-4908beb37202"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "job.attempt",
|
||||
"value": {
|
||||
"intValue": "1"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "cq34RtwooBc=",
|
||||
"parentSpanId": "AqkcRaZWl50=",
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164309031484554",
|
||||
"endTimeUnixNano": "1795164309038640365",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "PCx2LcCU6hc=",
|
||||
"parentSpanId": "AqkcRaZWl50=",
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164309108621018",
|
||||
"endTimeUnixNano": "1795164312013033788",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ReadTimeoutError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "cX5pEW5zHb0=",
|
||||
"parentSpanId": "AqkcRaZWl50=",
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164312013553635",
|
||||
"endTimeUnixNano": "1795164312029457465",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "yG0WSHlT7kQ=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164312924989622",
|
||||
"endTimeUnixNano": "1795164312936812921",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.36.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-worker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab6.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "742be3a8d93342da9117f92711193f78"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "ccEcqG6ZxPo=",
|
||||
"parentSpanId": "nBeLRAx4mXA=",
|
||||
"flags": 768,
|
||||
"name": "attachment process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"startTimeUnixNano": "1795164312924913496",
|
||||
"endTimeUnixNano": "1795164314747321271",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "messaging.system",
|
||||
"value": {
|
||||
"stringValue": "redis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "messaging.destination.name",
|
||||
"value": {
|
||||
"stringValue": "attachment-jobs"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "job.id",
|
||||
"value": {
|
||||
"stringValue": "9d711cdf-347d-4590-91da-4908beb37202"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "job.attempt",
|
||||
"value": {
|
||||
"intValue": "2"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "Qf833YZm6d8=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164312943026506",
|
||||
"endTimeUnixNano": "1795164314534106374",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "7QJGzOLYHEk=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.get_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314534253963",
|
||||
"endTimeUnixNano": "1795164314538495180",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "get_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "uj8SpqBZjXM=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314714992685",
|
||||
"endTimeUnixNano": "1795164314719977680",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "RPy3F6vP7SY=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314720094322",
|
||||
"endTimeUnixNano": "1795164314725722348",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "HvB6dqnveCs=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314725791217",
|
||||
"endTimeUnixNano": "1795164314727041344",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "DJqV5J+noKw=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314727130718",
|
||||
"endTimeUnixNano": "1795164314729233570",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "YhhL1bvMeTM=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314729272549",
|
||||
"endTimeUnixNano": "1795164314730313956",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "92rjoPWvm5g=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314730379104",
|
||||
"endTimeUnixNano": "1795164314732329971",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "7VymNXkyejE=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314732386665",
|
||||
"endTimeUnixNano": "1795164314733725616",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "5RGC0O8dQC8=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314733807464",
|
||||
"endTimeUnixNano": "1795164314736051098",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "EGXBlCQ2GhE=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314736092741",
|
||||
"endTimeUnixNano": "1795164314743975478",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "3Ryz4X3FR7yL0/lAM+Ofeg==",
|
||||
"spanId": "uADsu2p9U4s=",
|
||||
"parentSpanId": "ccEcqG6ZxPo=",
|
||||
"flags": 256,
|
||||
"name": "s3.delete_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164314744065993",
|
||||
"endTimeUnixNano": "1795164314746751040",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "delete_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"logs": [
|
||||
{
|
||||
"timestamp": "2026-11-20T08:45:12.027960+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "job_retry",
|
||||
"sampled": true,
|
||||
"trace_id": "dd1cb3e17dc547bc8bd3f94033e39f7a",
|
||||
"span_id": "717e69116e731dbd",
|
||||
"job_id": "9d711cdf-347d-4590-91da-4908beb37202",
|
||||
"attempt": 1,
|
||||
"error_code": "storage_unavailable"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
{
|
||||
"method": "Three repeats per full/10%/25% setting; each holds sessions lock until pg_locks confirms it. 3x503 plus one slow200 per repeat. Sampling is ordinary SDK head sampling without forced traceparent.",
|
||||
"runs": [
|
||||
{
|
||||
"mode": "full",
|
||||
"repeat": 0,
|
||||
"probes": [
|
||||
{
|
||||
"request_id": "95355e81cca54f83be9a19bd527e9825",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.5030926670000007,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "99c07cebfe6d37a278f4e0a9c603c1c2",
|
||||
"tempo_trace_found": true
|
||||
},
|
||||
{
|
||||
"request_id": "4e617be83d2247d0b849744fa0936deb",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.6241221669999995,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "806f738e3362110032113077494f8c8b",
|
||||
"tempo_trace_found": true
|
||||
},
|
||||
{
|
||||
"request_id": "b1c982a06765483eac67aa8cf72f2be1",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4874708749999996,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "a4f07c609d396e2071bb8bdbebcbad4f",
|
||||
"tempo_trace_found": true
|
||||
},
|
||||
{
|
||||
"request_id": "822110e4f137423084355e5c8d260dcb",
|
||||
"status": 200,
|
||||
"latency_seconds": 1.1728969580000026,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "8082c09a793e3535f33f203cb1dbe5da",
|
||||
"tempo_trace_found": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "full",
|
||||
"repeat": 1,
|
||||
"probes": [
|
||||
{
|
||||
"request_id": "da8b813d32d144cd844124a5fa86435e",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4984728329999975,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "0fe73d0ff759f1e9f2f6d9e304a8fb96",
|
||||
"tempo_trace_found": true
|
||||
},
|
||||
{
|
||||
"request_id": "afb0eccacc314dcc8d15e6375ed1f674",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.5027760410000006,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "04c97d704d04c30ed712dc1a6c506639",
|
||||
"tempo_trace_found": true
|
||||
},
|
||||
{
|
||||
"request_id": "2633bfaaa83040dc9ec3f21c968b6774",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.6313028749999994,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "a849e96eb0258348dba259183e9455b5",
|
||||
"tempo_trace_found": true
|
||||
},
|
||||
{
|
||||
"request_id": "5f1a8b638d594e7e8e249d83e29a3c0a",
|
||||
"status": 200,
|
||||
"latency_seconds": 1.1764352089999974,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "46f17923d4d83be8aec4863497913de9",
|
||||
"tempo_trace_found": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "full",
|
||||
"repeat": 2,
|
||||
"probes": [
|
||||
{
|
||||
"request_id": "f6cf9187274f4d4494f2546fdccf9cb5",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4932770829999953,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "b57747b1e2c746aafa5fb7d7e9ec7e33",
|
||||
"tempo_trace_found": true
|
||||
},
|
||||
{
|
||||
"request_id": "5820d13764fe4ab5826a24e6be1dd12c",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.5006451660000053,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "f167d8d9df62e009ab0c2c3ac4ee3fc3",
|
||||
"tempo_trace_found": true
|
||||
},
|
||||
{
|
||||
"request_id": "18e12c0c4a424141b916fd4f7c847c8b",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.5060892499999952,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "453c386a3fcd16e940832672ef0b072a",
|
||||
"tempo_trace_found": true
|
||||
},
|
||||
{
|
||||
"request_id": "56df38255f974d75a68b6defb7ae7dbf",
|
||||
"status": 200,
|
||||
"latency_seconds": 1.181737209000005,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "9380177eb739659ba6bfb122f4c5cb10",
|
||||
"tempo_trace_found": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "sample10",
|
||||
"repeat": 0,
|
||||
"probes": [
|
||||
{
|
||||
"request_id": "7c5131770a4e4ae78594065d30041f6b",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.5020687499999994,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "2f49be17fa517e268808bc7655dfec57",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "083b4e2c54b34fb695e3e097774a02fb",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.501452209000007,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "6b7b3c7eb4f9d107cdef92d247489d39",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "d63a94e59cc74918881a6c4821129fc0",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.5036532090000065,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "e6b3c5a1434815bc391a764eba18ad1f",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "7aba69c8960a4e58ab8e1189f91e8df8",
|
||||
"status": 200,
|
||||
"latency_seconds": 1.1895337499999954,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "a4b7c2e50c6db0db11b63d6988a5f581",
|
||||
"tempo_trace_found": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "sample10",
|
||||
"repeat": 1,
|
||||
"probes": [
|
||||
{
|
||||
"request_id": "db9a19fe6edf44b189e4c85527305b97",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.508617583000003,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "43478ce0a70fc97c1f551e0b770c8450",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "259f18e925b54205a7f86ccf23588581",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.6092364170000053,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "de05c2c1da9da6e5aaad96f7562f8626",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "7463cb267c1b4b7c836a79772d11d304",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4921612919999916,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "63f2de04875064912b355e6f7ccf2bf8",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "e6013482b7174a469182f9776832c5f8",
|
||||
"status": 200,
|
||||
"latency_seconds": 1.1786814999999962,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "bc385f8114ac1d41fa55f762830d9a70",
|
||||
"tempo_trace_found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "sample10",
|
||||
"repeat": 2,
|
||||
"probes": [
|
||||
{
|
||||
"request_id": "9a8de42292644772a11b19bb597b98e9",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4827315419999962,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "62ef4444e74027412566b41ff3330a09",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "444fd3c3d88c46cd89e1691852087fb3",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4932578330000013,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "95d7610acd0f35487ec10e503fc12622",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "6f616e3942134f7797c6c0b3767c0d0e",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.497054833000007,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "86793b238c9e2b7e586cb02e2f8bfc42",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "718f724c1a8d41bfa3ec720daee432d3",
|
||||
"status": 200,
|
||||
"latency_seconds": 1.1874764580000061,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "6b08c6339522e460bd77a9fd0f72438a",
|
||||
"tempo_trace_found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "sample25",
|
||||
"repeat": 0,
|
||||
"probes": [
|
||||
{
|
||||
"request_id": "5c805a3468b64739bd1c839f6ddedd56",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4879085000000032,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "5cf9d06acb989768f56a26c31e6d5255",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "ff5596ad44c6461589e621a7b3f09676",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.5042155419999972,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "f8f790abc18c384d58225dceb4d1e8a9",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "3ee8c8db9d4a4890b866d3cb40e18bb8",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4965452500000112,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "b85f2a7d3cd2ff3dd4fa66fc8b37dea6",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "f21c138335d64c688adb25c137c9a253",
|
||||
"status": 200,
|
||||
"latency_seconds": 1.1858503330000048,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "c1df2d7ecfb8448a2f33be6756f23c38",
|
||||
"tempo_trace_found": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "sample25",
|
||||
"repeat": 1,
|
||||
"probes": [
|
||||
{
|
||||
"request_id": "053ece24cfa1465d9702c67781863703",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.5010194580000018,
|
||||
"trace_sampled": true,
|
||||
"trace_id": "b7c73109f24a628720011f7401fc3d42",
|
||||
"tempo_trace_found": true
|
||||
},
|
||||
{
|
||||
"request_id": "7d30c3b85ebe480ea041d2261af90b3c",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4840848329999972,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "438e9ec5155e621e5ef24df2b6e12568",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "8c23bd6e2acc451d8abac7c787225b06",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4938515830000085,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "92267b13ab39964ae58d8bf054c2f52a",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "3ddb46344bae4d50989c8779264eeadf",
|
||||
"status": 200,
|
||||
"latency_seconds": 1.1738840829999901,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "66d1a465ebede54e734068bc6e6092ca",
|
||||
"tempo_trace_found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "sample25",
|
||||
"repeat": 2,
|
||||
"probes": [
|
||||
{
|
||||
"request_id": "dbcb7d27ff3a4067af49f2ad559beaf7",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.4999495829999887,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "88fa0b73db8d67b0be62f9ef653d5c1c",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "1644972bdbbf4ad4ae0bb1be96d60853",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.5033828749999998,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "815ba97d46bc980d6df03e8542f1bd94",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "34ecddeb28764f6da5aae311d8297f15",
|
||||
"status": 503,
|
||||
"latency_seconds": 1.486551832999993,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "5b1d99439c3aadf44db722c9543be3f3",
|
||||
"tempo_trace_found": false
|
||||
},
|
||||
{
|
||||
"request_id": "558612f66b374e6d964b927a866bd9bf",
|
||||
"status": 200,
|
||||
"latency_seconds": 1.1901341250000144,
|
||||
"trace_sampled": false,
|
||||
"trace_id": "e6505908ffe05caf83a8a7d49f4476a0",
|
||||
"tempo_trace_found": false
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"presence_verification": "Actual Tempo trace API checked for every probe: sampled present; unsampled HTTP404."
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"instance": "208b43d0e3fe4d8daeccd4d252d691fe", "container": "823e8f77a6bf447f87e3dcbd607f131734515b23ea33efaf00cb1e993f87770b", "running": false, "exit_code": 0, "seconds": 0.691}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"anonymous_object_and_list_403": true,
|
||||
"job_fields": [
|
||||
"attachment_id",
|
||||
"attempt",
|
||||
"created_at",
|
||||
"job_id",
|
||||
"pipeline_version",
|
||||
"schema_version",
|
||||
"traceparent",
|
||||
"type"
|
||||
],
|
||||
"job_bytes": 317,
|
||||
"no_source_bytes_or_credentials_in_envelope": true,
|
||||
"strict_invalid_envelope_checks": true
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"trace_id": "a630912396b64101b293cc306ee44ed8",
|
||||
"historical_span_count": 21,
|
||||
"historical_logs": 8,
|
||||
"fixed_metric_query_time": 1795164311.5720682,
|
||||
"metric_before": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795164311.572,
|
||||
"485"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metric_after": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795164311.572,
|
||||
"485"
|
||||
]
|
||||
}
|
||||
],
|
||||
"business_data_and_existing_session_survived": true,
|
||||
"all_telemetry_volumes_restarted": true,
|
||||
"trace_log_metric_history_survived": true
|
||||
}
|
||||
@@ -0,0 +1,886 @@
|
||||
{
|
||||
"trace_id": "a630912396b64101b293cc306ee44ed8",
|
||||
"request_id": "cb06618bb089444e8958f45beb2a8d1f",
|
||||
"status": "ready",
|
||||
"consumer_parent_matches_producer": true,
|
||||
"producer_descends_from_upload": true,
|
||||
"api_worker_logs_correlated": true,
|
||||
"secret_canaries_absent": true,
|
||||
"metric_labels_allowlisted": true,
|
||||
"trace": {
|
||||
"batches": [
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.36.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-dispatcher"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab6.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "6b019c7baab14c89976d2b4081f1c509"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "1X47WVGmwVA=",
|
||||
"parentSpanId": "XoCVQvOw3eM=",
|
||||
"flags": 768,
|
||||
"name": "attachment publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"startTimeUnixNano": "1795164306352857288",
|
||||
"endTimeUnixNano": "1795164306353855758",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "messaging.system",
|
||||
"value": {
|
||||
"stringValue": "redis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "messaging.destination.name",
|
||||
"value": {
|
||||
"stringValue": "attachment-jobs"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.36.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-worker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab6.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "c474b80aef4f42b1ba83aa52fc227420"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "jpUcOdrK9v8=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306353943301",
|
||||
"endTimeUnixNano": "1795164306358874754",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "uqFoOGdo/j8=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306361469707",
|
||||
"endTimeUnixNano": "1795164306363607190",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "Djfujh2+aLk=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.get_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306363643778",
|
||||
"endTimeUnixNano": "1795164306365315189",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "get_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "UqSD5a1Hm0M=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306502165196",
|
||||
"endTimeUnixNano": "1795164306504695870",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "FIWbTD1UmKU=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306504774140",
|
||||
"endTimeUnixNano": "1795164306509272099",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "sGmfWvZhAhE=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306509328359",
|
||||
"endTimeUnixNano": "1795164306510325785",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "cTvaoR1LX60=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306510386972",
|
||||
"endTimeUnixNano": "1795164306512247503",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "n7+0IIgeeOo=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306512289102",
|
||||
"endTimeUnixNano": "1795164306513273664",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "z5ewZlRmlSI=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306513330090",
|
||||
"endTimeUnixNano": "1795164306515021299",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "ThQkryD54tU=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306515051329",
|
||||
"endTimeUnixNano": "1795164306515871498",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "3F5tg5VjfcM=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306515917149",
|
||||
"endTimeUnixNano": "1795164306517590816",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "jRNKytkgcSc=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306517629032",
|
||||
"endTimeUnixNano": "1795164306523308482",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "EPrL86X8HKM=",
|
||||
"parentSpanId": "wUZSkS75RVs=",
|
||||
"flags": 256,
|
||||
"name": "s3.delete_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306523346489",
|
||||
"endTimeUnixNano": "1795164306524869630",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "delete_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "wUZSkS75RVs=",
|
||||
"parentSpanId": "1X47WVGmwVA=",
|
||||
"flags": 768,
|
||||
"name": "attachment process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"startTimeUnixNano": "1795164306353917740",
|
||||
"endTimeUnixNano": "1795164306525244943",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "messaging.system",
|
||||
"value": {
|
||||
"stringValue": "redis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "messaging.destination.name",
|
||||
"value": {
|
||||
"stringValue": "attachment-jobs"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "job.id",
|
||||
"value": {
|
||||
"stringValue": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "job.attempt",
|
||||
"value": {
|
||||
"intValue": "1"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.36.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab6.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "067e0d5099614580ab61516259b76c14"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "s72qs5x19wo=",
|
||||
"parentSpanId": "bWCElBq0RoM=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306279538296",
|
||||
"endTimeUnixNano": "1795164306279912941",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "9hTiABG5UiA=",
|
||||
"parentSpanId": "bWCElBq0RoM=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306279948610",
|
||||
"endTimeUnixNano": "1795164306280391627",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "PPdidexZwfM=",
|
||||
"parentSpanId": "bWCElBq0RoM=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306280710138",
|
||||
"endTimeUnixNano": "1795164306281834242",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "PKeSMnEZomk=",
|
||||
"parentSpanId": "XoCVQvOw3eM=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306282049715",
|
||||
"endTimeUnixNano": "1795164306285026998",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "XoCVQvOw3eM=",
|
||||
"parentSpanId": "bWCElBq0RoM=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795164306281855668",
|
||||
"endTimeUnixNano": "1795164306285749976",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "pjCRI5a2QQGyk8wwbuRO2A==",
|
||||
"spanId": "bWCElBq0RoM=",
|
||||
"parentSpanId": "ASNFZ4mrze8=",
|
||||
"flags": 768,
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"startTimeUnixNano": "1795164306279505300",
|
||||
"endTimeUnixNano": "1795164306285992974",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "http.request.method",
|
||||
"value": {
|
||||
"stringValue": "POST"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.route",
|
||||
"value": {
|
||||
"stringValue": "/api/v1/chats/{chat_id}/attachments"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.response.status_code",
|
||||
"value": {
|
||||
"intValue": "202"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"logs": [
|
||||
{
|
||||
"timestamp": "2026-11-20T08:45:06.285760+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-api",
|
||||
"event": "attachment_accepted",
|
||||
"sampled": true,
|
||||
"trace_id": "a630912396b64101b293cc306ee44ed8",
|
||||
"span_id": "6d6084941ab44683",
|
||||
"attachment_id": "a6e6251c-5774-4a3b-8424-c72d9568e4a6",
|
||||
"job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T08:45:06.525174+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "job_processed",
|
||||
"sampled": true,
|
||||
"trace_id": "a630912396b64101b293cc306ee44ed8",
|
||||
"span_id": "c14652912ef9455b",
|
||||
"job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2",
|
||||
"attempt": 1,
|
||||
"outcome": "ready"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T08:45:06.353780+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-dispatcher",
|
||||
"event": "job_published",
|
||||
"sampled": true,
|
||||
"trace_id": "a630912396b64101b293cc306ee44ed8",
|
||||
"span_id": "d57e3b5951a6c150",
|
||||
"job_id": "04e33bb9-f3eb-47ce-a968-9d5eb45a1ee2",
|
||||
"attempt": 1
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T08:45:06.285952+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-api",
|
||||
"event": "http_request",
|
||||
"sampled": true,
|
||||
"trace_id": "a630912396b64101b293cc306ee44ed8",
|
||||
"span_id": "6d6084941ab44683",
|
||||
"request_id": "cb06618bb089444e8958f45beb2a8d1f",
|
||||
"method": "POST",
|
||||
"route": "/api/v1/chats/{chat_id}/attachments",
|
||||
"status": 202,
|
||||
"duration_seconds": 0.006511
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T08:45:06.504597+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "operation_failed",
|
||||
"sampled": true,
|
||||
"trace_id": "a630912396b64101b293cc306ee44ed8",
|
||||
"span_id": "52a483e5ad479b43",
|
||||
"error_type": "ClientError"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T08:45:06.510258+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "operation_failed",
|
||||
"sampled": true,
|
||||
"trace_id": "a630912396b64101b293cc306ee44ed8",
|
||||
"span_id": "b0699f5af6610211",
|
||||
"error_type": "ClientError"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T08:45:06.513218+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "operation_failed",
|
||||
"sampled": true,
|
||||
"trace_id": "a630912396b64101b293cc306ee44ed8",
|
||||
"span_id": "9fbfb420881e78ea",
|
||||
"error_type": "ClientError"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T08:45:06.515824+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "operation_failed",
|
||||
"sampled": true,
|
||||
"trace_id": "a630912396b64101b293cc306ee44ed8",
|
||||
"span_id": "4e1424af20f9e2d5",
|
||||
"error_type": "ClientError"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 84 KiB |
@@ -0,0 +1,67 @@
|
||||
# Локальный runbook — lab6
|
||||
|
||||
Grafana: http://127.0.0.1:13006/d/messenger-lab6.
|
||||
Prometheus: http://127.0.0.1:19096.
|
||||
Loki и Tempo доступны через Explore и provisioned datasources.
|
||||
Не запрашивать credentials, HTTP bodies, raw URL или SQL parameters для диагностики.
|
||||
|
||||
## api-errors
|
||||
|
||||
1. Открыть API throughput, 5xx ratio и p95/p99. Health исключён.
|
||||
2. Проверить APIHighErrorRate в Prometheus /alerts: 5xx / все обработанные business requests за 30s > 5%, for 6s.
|
||||
3. В Loki: {service_name="messenger-api"} | json | status >= 500.
|
||||
4. Взять trace_id из лога и перейти в Tempo. X-Request-Id связывает ответ с тем же логом.
|
||||
5. Если основное время/ошибка в db.transaction, проверить db_pool_used/waiting и QueueObservationUnavailable:
|
||||
проблема на границе PostgreSQL/пула, а не декодирования изображения.
|
||||
6. Контролируемый инцидент теста удерживает ACCESS EXCLUSIVE lock таблицы sessions: lock_timeout=1.5s даёт 503.
|
||||
SELECT 1 readiness остаётся доступным; долгий span db.transaction показывает причину, даже если HTTP health зелёный.
|
||||
7. После снятия блокировки тот же Bearer снова работает. Подождать очистки 30s окна и for; проверить resolved.
|
||||
|
||||
## queue-lag
|
||||
|
||||
1. Проверить queue_depth и queue_oldest_age_seconds: это принятые незавершённые jobs в PostgreSQL,
|
||||
включая ещё не опубликованный outbox; это не длина Redis stream и не длительность resize.
|
||||
2. Если upload остаётся 202, а возраст/глубина растут, проверить worker busy/capacity и отсутствие новых job_processed.
|
||||
Удалённый процесс может оставаться в scrape cache до 30s; учитывать эту задержку.
|
||||
3. В Loki: {service_name=~"messenger-.*"} | json | event=~"attachment_accepted|job_published|job_processed|job_retry".
|
||||
4. Через trace_id пройти API → producer → consumer → S3/DB.
|
||||
Published без consumer указывает на отсутствие/недоступность worker или его чтения broker.
|
||||
Consumer с retry и ошибочным S3 span указывает на storage, а не потерю задания.
|
||||
5. Восстановить worker. Проверить ready, уменьшение backlog до нуля, исчезновение QueueLag.
|
||||
6. Не удалять volume, stream, job registry или staging ради зелёного dashboard.
|
||||
|
||||
## telemetry-backend
|
||||
|
||||
1. Если business API продолжает отвечать, но trace не находится, проверить Collector:
|
||||
otelcol_exporter_send_failed_spans_total, otelcol_exporter_queue_size/capacity.
|
||||
2. Уточнить, принимает ли Collector OTLP (otelcol_receiver_accepted_spans_total).
|
||||
Ошибки exporter при растущем accepted означают проблему downstream, а не отсутствие инструментации.
|
||||
3. Восстановить Tempo/Loki; проверить новый trace/log. Старые telemetry, отброшенные после 10s retry budget,
|
||||
не обязаны появиться. Бизнес-данные и telemetry имеют разные гарантии сохранности.
|
||||
4. Collector: memory limiter 192MiB, batch <=256, две exporter-очереди по 128 batches;
|
||||
SDK: span queue512/batch128, logs256/batch64, OTLP timeout1s.
|
||||
Нет бесконечной очереди и обязательной блокирующей синхронной отправки на HTTP-path.
|
||||
5. У каждого backend именованный volume. Отказ процесса и потеря диска — разные инциденты.
|
||||
|
||||
## SLI / SLO
|
||||
|
||||
Учебное окно 5 минут, при наличии запросов:
|
||||
- API availability SLI = 1 - sum(increase(http_server_requests_total{status_class="5xx"}[5m]))
|
||||
/ sum(increase(http_server_requests_total[5m])). Цель >=99%.
|
||||
4xx считаются корректно обслуженными запросами, health исключён.
|
||||
Это SLI обработанных API запросов: отказ LB/сети до API в знаменатель не попадает.
|
||||
- Time-to-ready latency SLI среди завершённых ready jobs:
|
||||
sum(increase(job_ready_duration_seconds_bucket{le="5"}[5m]))
|
||||
/ sum(increase(job_ready_duration_seconds_count[5m])). Цель >=95%.
|
||||
Не завершённые/failed jobs не скрываются: отдельно jobs_processed outcomes, job_retries и QueueLag.
|
||||
Этот latency SLO не является SLO доступности всего pipeline.
|
||||
|
||||
Короткие алерты 30s и 10s queue age предназначены для локальной демонстрации.
|
||||
Они не заявлены промышленными burn-rate alerts; выбран трек 1.
|
||||
|
||||
|
||||
Balanced default: TRACE_SAMPLE_RATIO=0.25; metrics/logs всегда включены.
|
||||
У несэмплированного запроса trace_id присутствует в log, но полного trace в Tempo не будет.
|
||||
Для контролируемого proof используется W3C sampled=1; для полной локальной диагностики:
|
||||
TRACE_SAMPLE_RATIO=1 docker compose up -d --no-deps api worker dispatcher.
|
||||
Это не tail sampling: ошибки и медленные spans не получают автоматического приоритета.
|
||||
@@ -0,0 +1,27 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
rules:
|
||||
- alert: APIHighErrorRate
|
||||
expr: sum(rate(http_server_requests_total{service_name="messenger-api",status_class="5xx"}[30s])) / clamp_min(sum(rate(http_server_requests_total{service_name="messenger-api"}[30s])), 0.001) > 0.05
|
||||
for: 6s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: API 5xx share exceeds 5 percent in the last 30 seconds
|
||||
runbook: infra/RUNBOOK.md#api-errors
|
||||
- alert: QueueLag
|
||||
expr: max(queue_oldest_age_seconds{service_name="messenger-dispatcher"}) > 10 and max(queue_depth{service_name="messenger-dispatcher"}) > 0
|
||||
for: 6s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Accepted unfinished image work older than 10 seconds
|
||||
runbook: infra/RUNBOOK.md#queue-lag
|
||||
- alert: QueueObservationUnavailable
|
||||
expr: min(queue_observation_ok{service_name="messenger-dispatcher"}) == 0
|
||||
for: 6s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Dispatcher cannot observe PostgreSQL backlog
|
||||
runbook: infra/RUNBOOK.md#api-errors
|
||||
@@ -0,0 +1,713 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "MTUSI lab6 — Messenger observability",
|
||||
"schemaVersion": 41,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"tags": [
|
||||
"mtusi",
|
||||
"lab6"
|
||||
],
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API throughput (health excluded)",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(http_server_requests_total{service_name=\"messenger-api\"}[1m]))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API 5xx ratio",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(http_server_requests_total{service_name=\"messenger-api\",status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{service_name=\"messenger-api\"}[1m])),0.001)",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API p50 from histogram",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "API p95 from histogram",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "API p99 from histogram",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Worker jobs by outcome",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (rate(jobs_processed_total{service_name=\"messenger-worker\"}[1m]))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Worker processing p95",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Accepted to ready p95",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(job_ready_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Accepted unfinished jobs",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(queue_depth{service_name=\"messenger-dispatcher\"})",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Oldest unfinished job age",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(queue_oldest_age_seconds{service_name=\"messenger-dispatcher\"})",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "DB pool used",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(service_name) (db_pool_used{service_name=\"messenger-api\"})",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "DB pool waiting",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(service_name) (db_pool_waiting{service_name=\"messenger-api\"})",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "Worker busy / capacity",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Process resident memory",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "process_resident_memory_bytes{service_name=~\"messenger-.*\"}",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Process CPU seconds per second",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(service_name)(rate(process_cpu_seconds_total{service_name=~\"messenger-.*\"}[1m]))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Container memory incl renderer",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "container_memory_bytes{service_name=~\"messenger-.*\"}",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Local alerts",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "ALERTS{alertstate=\"firing\"}",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"title": "Collector failed span exports",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 100,
|
||||
"title": "Correlated structured logs: open TraceID for API / publish / worker chain",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 72,
|
||||
"w": 24,
|
||||
"h": 10
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=~\"messenger-.*\"} | json",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"sortOrder": "Descending"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 101,
|
||||
"title": "Terminal failed jobs",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 82,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(job_retries_total{service_name=\"messenger-worker\",outcome=\"failed\"}[1m]))"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 102,
|
||||
"title": "Retry attempts",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 82,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(job_retries_total{service_name=\"messenger-worker\",outcome=\"retry\"}[1m]))"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Messenger
|
||||
folder: MTUSI
|
||||
type: file
|
||||
updateIntervalSeconds: 10
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,32 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
datasourceUid: tempo
|
||||
matcherRegex: '"trace_id":"([a-f0-9]{32})"'
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: '-1m'
|
||||
spanEndTimeShift: '1m'
|
||||
filterByTraceID: false
|
||||
customQuery: true
|
||||
query: '{service_name=~"messenger-.*"} | json | trace_id="$${__trace.traceId}"'
|
||||
@@ -0,0 +1,46 @@
|
||||
global
|
||||
maxconn 1024
|
||||
nbthread 1
|
||||
hard-stop-after 10s
|
||||
|
||||
defaults
|
||||
mode http
|
||||
timeout connect 500ms
|
||||
timeout check 500ms
|
||||
timeout queue 1s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 3s
|
||||
retries 2
|
||||
retry-on conn-failure
|
||||
option http-server-close
|
||||
|
||||
resolvers docker
|
||||
nameserver docker_dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 2
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold nx 1s
|
||||
hold refused 1s
|
||||
hold timeout 1s
|
||||
hold other 1s
|
||||
hold obsolete 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend replicas
|
||||
|
||||
backend replicas
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server inter 1s fall 2 rise 2
|
||||
server-template api 1-4 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
|
||||
|
||||
listen metrics
|
||||
bind :8404
|
||||
stats enable
|
||||
stats uri /stats
|
||||
@@ -0,0 +1,33 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 24h
|
||||
ingestion_rate_mb: 4
|
||||
ingestion_burst_size_mb: 8
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,66 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
metric_expiration: 30s
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 128
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 3s
|
||||
max_elapsed_time: 10s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 128
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 3s
|
||||
max_elapsed_time: 10s
|
||||
service:
|
||||
telemetry:
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,11 @@
|
||||
global:
|
||||
scrape_interval: 2s
|
||||
evaluation_interval: 2s
|
||||
rule_files: [/etc/prometheus/alerts.yaml]
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
static_configs:
|
||||
- targets: [otel-collector:8889]
|
||||
- job_name: collector
|
||||
static_configs:
|
||||
- targets: [otel-collector:8888]
|
||||
@@ -0,0 +1,24 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
ingester:
|
||||
trace_idle_period: 5s
|
||||
max_block_duration: 1m
|
||||
max_block_bytes: 1048576
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 24h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,42 @@
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY
|
||||
, username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$')
|
||||
, display_name TEXT NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
, password_hash TEXT NOT NULL
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
;
|
||||
CREATE TABLE chats (
|
||||
id UUID PRIMARY KEY
|
||||
, title TEXT NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, last_message_at TIMESTAMPTZ
|
||||
)
|
||||
;
|
||||
CREATE TABLE chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id)
|
||||
, user_id UUID NOT NULL REFERENCES users(id)
|
||||
, PRIMARY KEY (chat_id, user_id)
|
||||
)
|
||||
;
|
||||
CREATE INDEX chat_members_user ON chat_members (user_id, chat_id)
|
||||
;
|
||||
CREATE TABLE messages (
|
||||
id UUID PRIMARY KEY
|
||||
, chat_id UUID NOT NULL REFERENCES chats(id)
|
||||
, sender_id UUID NOT NULL REFERENCES users(id)
|
||||
, text TEXT NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000)
|
||||
, created_at TIMESTAMPTZ NOT NULL
|
||||
, FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
)
|
||||
;
|
||||
CREATE INDEX messages_history ON messages (chat_id, created_at, id)
|
||||
;
|
||||
CREATE TABLE sessions (
|
||||
token_hash TEXT PRIMARY KEY
|
||||
, user_id UUID NOT NULL REFERENCES users(id)
|
||||
, expires_at TIMESTAMPTZ NOT NULL
|
||||
)
|
||||
;
|
||||
CREATE INDEX sessions_expiry ON sessions (expires_at)
|
||||
;
|
||||
@@ -0,0 +1,17 @@
|
||||
-- PostgreSQL TEXT cannot contain U+0000. Store validated UTF-8 losslessly in BYTEA.
|
||||
-- API validation keeps the contract's code-point limits; these bounds cap bytes.
|
||||
ALTER TABLE users
|
||||
DROP CONSTRAINT users_display_name_check
|
||||
, ALTER COLUMN display_name TYPE BYTEA USING convert_to(display_name, 'UTF8')
|
||||
, ADD CONSTRAINT users_display_name_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400)
|
||||
;
|
||||
ALTER TABLE chats
|
||||
DROP CONSTRAINT chats_title_check
|
||||
, ALTER COLUMN title TYPE BYTEA USING convert_to(title, 'UTF8')
|
||||
, ADD CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400)
|
||||
;
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_check
|
||||
, ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8')
|
||||
, ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000)
|
||||
;
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users(id)
|
||||
, chat_id UUID NOT NULL REFERENCES chats(id)
|
||||
, key VARCHAR(128) NOT NULL
|
||||
, request_hash CHAR(64) NOT NULL
|
||||
, message_id UUID REFERENCES messages(id)
|
||||
, expires_at TIMESTAMPTZ NOT NULL
|
||||
, PRIMARY KEY (user_id, chat_id, key)
|
||||
)
|
||||
;
|
||||
CREATE INDEX idempotency_expiry ON idempotency_keys (expires_at)
|
||||
;
|
||||
@@ -0,0 +1,30 @@
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_bytes
|
||||
, ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000)
|
||||
, ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}'
|
||||
;
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY
|
||||
, chat_id UUID NOT NULL REFERENCES chats(id)
|
||||
, owner_id UUID NOT NULL REFERENCES users(id)
|
||||
, filename TEXT NOT NULL
|
||||
, object_key TEXT NOT NULL UNIQUE
|
||||
, content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg'))
|
||||
, size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760)
|
||||
, sha256 CHAR(64) NOT NULL
|
||||
, state TEXT NOT NULL CHECK (state IN ('uploading', 'ready', 'storage_missing'))
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id)
|
||||
)
|
||||
;
|
||||
CREATE INDEX attachments_chat ON attachments (chat_id, id)
|
||||
;
|
||||
CREATE TABLE message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id)
|
||||
, attachment_id UUID NOT NULL REFERENCES attachments(id)
|
||||
, PRIMARY KEY (message_id, attachment_id)
|
||||
)
|
||||
;
|
||||
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id)
|
||||
;
|
||||
@@ -0,0 +1,40 @@
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT attachments_state_check
|
||||
, ADD CONSTRAINT attachments_state_check CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed', 'storage_missing'))
|
||||
, ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1
|
||||
, ADD COLUMN staging_key TEXT
|
||||
, ADD COLUMN variants JSONB NOT NULL DEFAULT '[]'
|
||||
, ADD COLUMN error_code TEXT
|
||||
, ADD COLUMN current_job_id UUID
|
||||
;
|
||||
CREATE TABLE attachment_jobs (
|
||||
id UUID PRIMARY KEY
|
||||
, attachment_id UUID NOT NULL REFERENCES attachments(id)
|
||||
, pipeline_version INTEGER NOT NULL
|
||||
, source_key TEXT NOT NULL
|
||||
, state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded'))
|
||||
, attempt INTEGER NOT NULL DEFAULT 0
|
||||
, max_attempts INTEGER NOT NULL DEFAULT 5
|
||||
, lease_token UUID
|
||||
, lease_until TIMESTAMPTZ
|
||||
, next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, published_at TIMESTAMPTZ
|
||||
, error_code TEXT
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, UNIQUE (attachment_id, pipeline_version)
|
||||
)
|
||||
;
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_current_job FOREIGN KEY (current_job_id) REFERENCES attachment_jobs(id)
|
||||
;
|
||||
CREATE INDEX jobs_due ON attachment_jobs (state, next_attempt_at)
|
||||
;
|
||||
CREATE TABLE job_events (
|
||||
id BIGSERIAL PRIMARY KEY
|
||||
, job_id UUID NOT NULL REFERENCES attachment_jobs(id)
|
||||
, event TEXT NOT NULL
|
||||
, attempt INTEGER NOT NULL
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
;
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD COLUMN trace_context JSONB NOT NULL DEFAULT '{}'
|
||||
;
|
||||
@@ -0,0 +1,27 @@
|
||||
boto3==1.40.61
|
||||
botocore==1.40.76
|
||||
certifi==2026.10.23
|
||||
charset-normalizer==3.5.2
|
||||
googleapis-common-protos==1.76.0
|
||||
idna==3.20
|
||||
importlib_metadata==8.7.1
|
||||
jmespath==1.1.0
|
||||
opentelemetry-api==1.36.0
|
||||
opentelemetry-exporter-otlp-proto-common==1.36.0
|
||||
opentelemetry-exporter-otlp-proto-http==1.36.0
|
||||
opentelemetry-proto==1.36.0
|
||||
opentelemetry-sdk==1.36.0
|
||||
opentelemetry-semantic-conventions==0.57b0
|
||||
pillow==13.0.0
|
||||
protobuf==6.33.6
|
||||
psycopg==3.2.3
|
||||
psycopg-binary==3.2.3
|
||||
psycopg-pool==3.2.4
|
||||
python-dateutil==2.9.0.post0
|
||||
redis==5.2.1
|
||||
requests==2.34.2
|
||||
s3transfer==0.14.0
|
||||
six==1.17.0
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
zipp==4.1.2
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local credentials without printing or overwriting existing values."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / 'secrets'
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
|
||||
def save(name, value):
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value)
|
||||
path.chmod(0o644) # UID 10001 in Compose; parent is owner-only 0700.
|
||||
return path.read_text().strip()
|
||||
|
||||
password = save('postgres_password', secrets.token_hex(24))
|
||||
save('database_url', 'postgresql://messenger:' + password + '@postgres:5432/messenger')
|
||||
save('cache_url', 'redis://redis:6379/0')
|
||||
save('broker_url', 'redis://broker:6379/0')
|
||||
identities = []
|
||||
for role, actions in {
|
||||
'admin': ['Admin', 'Read', 'Write', 'List'],
|
||||
'api': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'],
|
||||
'worker': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'],
|
||||
}.items():
|
||||
key = save('s3_' + role + '_key', secrets.token_hex(12))
|
||||
secret = save('s3_' + role + '_secret', secrets.token_hex(24))
|
||||
identities.append({'name': role, 'credentials': [{'accessKey': key, 'secretKey': secret}], 'actions': actions})
|
||||
configuration = directory / 's3_config'
|
||||
configuration.write_text(json.dumps({'identities': identities}))
|
||||
configuration.chmod(0o600) # Only the root S3 gateway mounts the full identity configuration.
|
||||
env = root / '.env'
|
||||
if not env.exists():
|
||||
env.write_text((root / '.env.example').read_text())
|
||||
env.chmod(0o600)
|
||||
print('Local configuration ready; no secrets printed.')
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
python3 scripts/snapshot.py > evidence/restart-before.json
|
||||
docker compose stop api lb worker dispatcher
|
||||
docker compose restart postgres redis s3-store s3 broker otel-collector prometheus loki tempo grafana
|
||||
docker compose up -d --wait postgres redis s3-store s3 broker otel-collector prometheus loki tempo grafana
|
||||
docker compose up -d --wait api lb worker dispatcher
|
||||
python3 scripts/snapshot.py > evidence/restart-after.json
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Sanitized process and resource evidence; never dump container environments."""
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split()
|
||||
records = []
|
||||
for container in ids:
|
||||
raw = subprocess.check_output(['docker', 'inspect', container], text=True)
|
||||
data = json.loads(raw)[0]
|
||||
records.append({'container': data['Id'], 'name': data['Name'], 'service': data['Config']['Labels']['com.docker.compose.service'], 'image': data['Config']['Image'], 'user': data['Config']['User'], 'running': data['State']['Running'], 'started_at': data['State']['StartedAt'], 'exit_code': data['State']['ExitCode'], 'oom_killed': data['State']['OOMKilled'], 'memory_limit': data['HostConfig']['Memory'], 'nano_cpus': data['HostConfig']['NanoCpus'], 'ports': data['NetworkSettings']['Ports'], 'mounts': [{'type': m['Type'], 'destination': m['Destination'], 'name': m.get('Name')} for m in data['Mounts']]})
|
||||
print(json.dumps(records, indent=2))
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stop_one.py
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Resolve the requested process alias using each actual container, never a fake header."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(['docker', *args], text=True).strip()
|
||||
|
||||
|
||||
def instance(container):
|
||||
code = "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=1).headers['X-Instance-Id'])"
|
||||
return docker('exec', container, 'python', '-c', code)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
target = os.environ['TARGET_INSTANCE_ID']
|
||||
started = time.monotonic()
|
||||
for container in docker('compose', 'ps', '-q', 'api').splitlines():
|
||||
if instance(container) == target:
|
||||
docker('stop', '--time', '8', container)
|
||||
state = json.loads(docker('inspect', '--format', '{{json .State}}', container))
|
||||
assert not state['Running'], 'target must remain stopped'
|
||||
record = {'instance': target, 'container': container, 'running': state['Running'], 'exit_code': state['ExitCode'], 'seconds': round(time.monotonic() - started, 3)}
|
||||
with Path('evidence/stopped-instances.jsonl').open('a') as out:
|
||||
out.write(json.dumps(record) + '\n')
|
||||
print(json.dumps(record))
|
||||
break
|
||||
else:
|
||||
raise SystemExit('Requested live API instance not found')
|
||||
@@ -0,0 +1,16 @@
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
base = os.environ.get('BASE_URL', 'http://127.0.0.1:18087')
|
||||
for attempt in range(120):
|
||||
try:
|
||||
with urllib.request.urlopen(base + '/health/ready', timeout=2) as response:
|
||||
if response.status == 200:
|
||||
print('Ready through load balancer.')
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
raise SystemExit('Readiness timeout')
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
"""Explicit local operator tools; no administrative HTTP endpoint."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
from pipeline import connect, audit
|
||||
from objects import Objects
|
||||
from postgres import setting
|
||||
|
||||
|
||||
def replay(job_id):
|
||||
with connect() as conn, conn.transaction():
|
||||
job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job_id,)).fetchone()
|
||||
if job is None or job['state'] != 'failed':
|
||||
raise ValueError('Replay requires a failed job')
|
||||
attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND current_job_id = %s FOR UPDATE', (job_id,)).fetchone()
|
||||
if attachment is None or attachment['state'] != 'failed' or Objects(setting).head(job['source_key']) is None:
|
||||
raise ValueError('Current failed source is unavailable')
|
||||
conn.execute('''
|
||||
UPDATE attachment_jobs
|
||||
SET state = 'queued'
|
||||
, max_attempts = attempt + %s
|
||||
, published_at = NULL
|
||||
, next_attempt_at = clock_timestamp()
|
||||
, error_code = NULL
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
''', (int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), job_id))
|
||||
conn.execute("UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job_id,))
|
||||
audit(conn, job, 'operator_replay')
|
||||
return {'job_id': str(job_id), 'replayed': True}
|
||||
|
||||
|
||||
def version(attachment_id, number):
|
||||
if number not in (1, 2):
|
||||
raise ValueError('Only implemented pipeline versions 1 and 2 are supported')
|
||||
with connect() as conn, conn.transaction():
|
||||
row = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone()
|
||||
if row is None or number != row['pipeline_version'] + 1 or row['state'] in ('uploading', 'failed', 'storage_missing'):
|
||||
raise ValueError('Version must advance the current process exactly once')
|
||||
old = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (row['current_job_id'],)).fetchone()
|
||||
source = row['object_key'] if row['state'] == 'ready' else old['source_key']
|
||||
if Objects(setting).head(source) is None:
|
||||
raise ValueError('Source unavailable')
|
||||
conn.execute("UPDATE attachment_jobs SET state = 'superseded', lease_token = NULL, lease_until = NULL WHERE 1=1 AND id = %s", (old['id'],))
|
||||
audit(conn, old, 'operator_supersede')
|
||||
job_id = uuid.uuid4()
|
||||
conn.execute('''
|
||||
INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
''', (job_id, attachment_id, number, source, int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))))
|
||||
conn.execute('''
|
||||
UPDATE attachments
|
||||
SET pipeline_version = %s
|
||||
, current_job_id = %s
|
||||
, state = 'queued'
|
||||
, variants = '[]'
|
||||
, error_code = NULL
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
''', (number, job_id, attachment_id))
|
||||
return {'job_id': str(job_id), 'version': number}
|
||||
|
||||
|
||||
def cleanup(apply=False, retention=86400):
|
||||
if retention < 86400:
|
||||
raise ValueError('Retention must be >=24 hours')
|
||||
objects = Objects(setting)
|
||||
counts = {'ready_staging': 0, 'expired_staging': 0}
|
||||
with connect() as conn:
|
||||
# Intent rows cover every valid staging PUT. Never collect active jobs.
|
||||
last = '00000000-0000-0000-0000-000000000000'
|
||||
while True:
|
||||
rows = conn.execute('SELECT id FROM attachments WHERE 1=1 AND id > %s ORDER BY id LIMIT 100', (last,)).fetchall()
|
||||
if not rows:
|
||||
break
|
||||
for candidate in rows:
|
||||
last = candidate['id']
|
||||
with conn.transaction():
|
||||
row = conn.execute('''
|
||||
SELECT *
|
||||
FROM attachments
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
AND (state = 'ready' OR (state IN ('uploading', 'failed') AND updated_at < clock_timestamp() - %s * INTERVAL '1 second'))
|
||||
FOR UPDATE SKIP LOCKED
|
||||
''', (last, retention)).fetchone()
|
||||
if row is None or not row['staging_key'] or not row['staging_key'].startswith('staging/'):
|
||||
continue
|
||||
counts['ready_staging' if row['state'] == 'ready' else 'expired_staging'] += 1
|
||||
if apply:
|
||||
objects.delete(row['staging_key'])
|
||||
if row['state'] == 'uploading' and row['current_job_id'] is None:
|
||||
conn.execute('DELETE FROM attachments WHERE 1=1 AND id = %s', (last,))
|
||||
else:
|
||||
conn.execute('UPDATE attachments SET staging_key = NULL WHERE 1=1 AND id = %s', (last,))
|
||||
return dict(apply=apply, retention_seconds=retention, counts=counts)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('action', choices=['failed', 'replay', 'version', 'cleanup'])
|
||||
p.add_argument('identifier', nargs='?')
|
||||
p.add_argument('--version', type=int)
|
||||
p.add_argument('--apply', action='store_true')
|
||||
args = p.parse_args()
|
||||
try:
|
||||
if args.action == 'failed':
|
||||
with connect() as conn:
|
||||
result = conn.execute("SELECT id, attachment_id, pipeline_version, attempt, max_attempts, error_code FROM attachment_jobs WHERE 1=1 AND state = 'failed' ORDER BY created_at LIMIT 100").fetchall()
|
||||
elif args.action == 'replay':
|
||||
result = replay(uuid.UUID(args.identifier))
|
||||
elif args.action == 'version':
|
||||
result = version(uuid.UUID(args.identifier), args.version)
|
||||
else:
|
||||
result = cleanup(args.apply)
|
||||
print(json.dumps(result, default=str, indent=2))
|
||||
except Exception:
|
||||
print(json.dumps({'error': 'Operator action refused or dependency unavailable; no secrets logged.'}))
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Idempotent private bucket bootstrap, executed only with administrative S3 keys."""
|
||||
from botocore.exceptions import ClientError
|
||||
from objects import Objects
|
||||
from postgres import setting
|
||||
|
||||
objects = Objects(setting)
|
||||
try:
|
||||
objects.client.create_bucket(Bucket=objects.bucket)
|
||||
except ClientError as exc:
|
||||
if exc.response['Error']['Code'] not in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
|
||||
raise RuntimeError('Bucket bootstrap failed') from None
|
||||
objects.ready()
|
||||
print('Private bucket ready; static authenticated identities loaded from local secret config.')
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Shared HTTP contract; storage is supplied by each laboratory."""
|
||||
import base64
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status, code, message):
|
||||
self.status, self.code, self.message = status, code, message
|
||||
|
||||
|
||||
def invalid():
|
||||
raise APIError(400, 'invalid_request', 'Неверные параметры запроса')
|
||||
|
||||
|
||||
def unauthorized():
|
||||
raise APIError(401, 'unauthorized', 'Требуется действительная идентификация')
|
||||
|
||||
|
||||
def missing():
|
||||
raise APIError(404, 'not_found', 'Объект не найден')
|
||||
|
||||
|
||||
def identifier(value):
|
||||
try:
|
||||
if not isinstance(value, str):
|
||||
invalid()
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
invalid()
|
||||
|
||||
|
||||
def text(value, maximum, minimum=1):
|
||||
if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
|
||||
invalid()
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
def fields(value, required):
|
||||
if not isinstance(value, dict) or set(value) != set(required):
|
||||
invalid()
|
||||
|
||||
|
||||
def username(value):
|
||||
if not isinstance(value, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', value):
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
def now():
|
||||
return dt.datetime.now(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def page_args(query, scope):
|
||||
if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()):
|
||||
invalid()
|
||||
limit = query.get('limit', ['50'])[0]
|
||||
if not re.fullmatch(r'[0-9]+', limit):
|
||||
invalid()
|
||||
try:
|
||||
limit = int(limit)
|
||||
except ValueError:
|
||||
invalid()
|
||||
if not 1 <= limit <= 100:
|
||||
invalid()
|
||||
cursor = None
|
||||
if 'cursor' in query:
|
||||
raw = query['cursor'][0]
|
||||
try:
|
||||
if not raw or len(raw) > 1024:
|
||||
invalid()
|
||||
decoded = json.loads(base64.b64decode(raw + '=' * (-len(raw) % 4), altchars=b'-_', validate=True))
|
||||
if not isinstance(decoded, list) or len(decoded) != 3 or decoded[0] != scope:
|
||||
invalid()
|
||||
stamp = decoded[1]
|
||||
if not isinstance(stamp, str) or not stamp.endswith('Z'):
|
||||
invalid()
|
||||
stamp = dt.datetime.fromisoformat(stamp[:-1] + '+00:00').isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
cursor = (stamp, identifier(decoded[2]))
|
||||
except (ValueError, TypeError, UnicodeError):
|
||||
invalid()
|
||||
return limit, cursor
|
||||
|
||||
|
||||
def page(items, limit, scope):
|
||||
more = len(items) > limit
|
||||
items = items[:limit]
|
||||
cursor = None
|
||||
if more:
|
||||
last = items[-1]
|
||||
cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=')
|
||||
return {'items': items, 'next_cursor': cursor}
|
||||
@@ -0,0 +1,13 @@
|
||||
"""One-shot migration job; replicas never execute DDL."""
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
from postgres import setting
|
||||
|
||||
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=5) as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(2301003)')
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)')
|
||||
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
|
||||
if not conn.execute('SELECT version FROM schema_migrations WHERE 1=1 AND version = %s', (path.name,)).fetchone():
|
||||
conn.execute(path.read_text())
|
||||
conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.name,))
|
||||
print('Applied', path.name)
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
"""Bounded S3 operations. No root credentials or filename-derived object keys."""
|
||||
import contextlib
|
||||
import telemetry as tele
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
import warnings
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from PIL import Image, PngImagePlugin, UnidentifiedImageError
|
||||
|
||||
from common import APIError, invalid
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024
|
||||
PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
|
||||
warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
|
||||
UPLOAD_SLOTS = threading.BoundedSemaphore(8)
|
||||
DECODE_SLOTS = threading.BoundedSemaphore(1)
|
||||
|
||||
|
||||
def unavailable():
|
||||
raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно')
|
||||
|
||||
|
||||
def filename(value):
|
||||
if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip():
|
||||
invalid()
|
||||
if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value):
|
||||
invalid()
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def bounded_upload():
|
||||
if not UPLOAD_SLOTS.acquire(blocking=False):
|
||||
raise APIError(503, 'unavailable', 'Все слоты загрузки заняты')
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
UPLOAD_SLOTS.release()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def receive(handler):
|
||||
lengths = handler.headers.get_all('Content-Length', [])
|
||||
if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'):
|
||||
invalid()
|
||||
try:
|
||||
length = int(lengths[0])
|
||||
except ValueError:
|
||||
invalid()
|
||||
if length > MAX_BYTES:
|
||||
raise APIError(413, 'payload_too_large', 'Максимум 10 MiB')
|
||||
if length <= 0:
|
||||
invalid()
|
||||
mime = handler.headers.get_content_type()
|
||||
if mime not in ('image/png', 'image/jpeg'):
|
||||
raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG')
|
||||
deadline = time.monotonic() + 15
|
||||
digest = hashlib.sha256()
|
||||
# Ephemeral bounded spool is never a durable source of a confirmed attachment.
|
||||
with tempfile.TemporaryFile(mode='w+b') as source:
|
||||
remaining = length
|
||||
while remaining:
|
||||
handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic())))
|
||||
if time.monotonic() >= deadline:
|
||||
invalid()
|
||||
try:
|
||||
chunk = handler.rfile.read1(min(64 * 1024, remaining))
|
||||
except (OSError, TimeoutError):
|
||||
invalid()
|
||||
if not chunk:
|
||||
invalid()
|
||||
digest.update(chunk)
|
||||
source.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
if not DECODE_SLOTS.acquire(timeout=10):
|
||||
raise APIError(503, 'unavailable', 'Проверка изображений перегружена')
|
||||
try:
|
||||
source.flush()
|
||||
for decode in (False,):
|
||||
source.seek(0)
|
||||
# Pillow.close releases pixel buffers and may close its input file.
|
||||
# A duplicate descriptor preserves the original for the S3 PUT.
|
||||
with os.fdopen(os.dup(source.fileno()), 'rb') as image_file:
|
||||
image = Image.open(image_file, formats=['PNG', 'JPEG'])
|
||||
try:
|
||||
if image.width * image.height > MAX_PIXELS:
|
||||
raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей')
|
||||
if Image.MIME[image.format] != mime:
|
||||
raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению')
|
||||
if decode:
|
||||
image.load()
|
||||
else:
|
||||
image.verify()
|
||||
finally:
|
||||
image.close() # Free pixels BEFORE releasing the decoder semaphore.
|
||||
except (Image.DecompressionBombWarning, Image.DecompressionBombError):
|
||||
raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None
|
||||
finally:
|
||||
DECODE_SLOTS.release()
|
||||
source.seek(0)
|
||||
yield source, length, mime, digest.hexdigest()
|
||||
|
||||
|
||||
class Objects:
|
||||
def __init__(self, setting):
|
||||
self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
||||
self.client = boto3.client('s3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:8333'), region_name='us-east-1', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required'))
|
||||
self.client = tele.S3Client(self.client)
|
||||
|
||||
def head(self, key):
|
||||
try:
|
||||
return self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'):
|
||||
return None
|
||||
unavailable()
|
||||
except BotoCoreError:
|
||||
unavailable()
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
except (ClientError, BotoCoreError):
|
||||
unavailable()
|
||||
|
||||
def put(self, key, source, length, mime, digest):
|
||||
try:
|
||||
self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest})
|
||||
except (ClientError, BotoCoreError):
|
||||
unavailable()
|
||||
|
||||
def get(self, key):
|
||||
try:
|
||||
return self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
except (ClientError, BotoCoreError):
|
||||
unavailable()
|
||||
|
||||
def delete(self, key):
|
||||
try:
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
except (ClientError, BotoCoreError):
|
||||
unavailable()
|
||||
+393
@@ -0,0 +1,393 @@
|
||||
"""Durable outbox, Redis Streams delivery, bounded worker and fenced result commits."""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import telemetry as tele
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
import redis
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
from common import APIError
|
||||
from objects import Objects, MAX_BYTES
|
||||
from postgres import setting
|
||||
|
||||
STREAM, GROUP = 'attachment-jobs', 'image-workers'
|
||||
LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '60'))
|
||||
STOP = threading.Event()
|
||||
|
||||
|
||||
def raw_connect():
|
||||
return psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row, autocommit=True, connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1500')
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def connect():
|
||||
with tele.span('db.transaction', {'db.system':'postgresql'}, tele.SpanKind.CLIENT):
|
||||
with raw_connect() as connection:
|
||||
yield connection
|
||||
|
||||
|
||||
def broker():
|
||||
client = redis.Redis.from_url(setting('BROKER_URL'), decode_responses=True, socket_connect_timeout=1, socket_timeout=3)
|
||||
try:
|
||||
client.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
|
||||
except redis.ResponseError as error:
|
||||
if 'BUSYGROUP' not in str(error):
|
||||
raise
|
||||
return client
|
||||
|
||||
|
||||
def event(job):
|
||||
return {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], 'attempt': job['attempt'] + 1, 'created_at': job['created_at'].isoformat().replace('+00:00', 'Z'), **(job.get('trace_context') or {})}
|
||||
|
||||
|
||||
def validate(message):
|
||||
required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
|
||||
if not isinstance(message, dict) or set(message) - required - {'traceparent', 'tracestate'} or not required <= set(message):
|
||||
return False
|
||||
try:
|
||||
uuid.UUID(message['job_id']); uuid.UUID(message['attachment_id'])
|
||||
stamp = dt.datetime.fromisoformat(message['created_at'].replace('Z', '+00:00'))
|
||||
if stamp.tzinfo is None or not message['created_at'].endswith('Z'):
|
||||
return False
|
||||
if 'traceparent' in message and (not isinstance(message['traceparent'], str) or not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', message['traceparent'])):
|
||||
return False
|
||||
if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512):
|
||||
return False
|
||||
return type(message['schema_version']) is int and message['schema_version'] == 1 and message['type'] == 'attachment.process.v1' and all(type(message[k]) is int and message[k] >= 1 for k in ['pipeline_version', 'attempt'])
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
return False
|
||||
|
||||
|
||||
def audit(conn, job, name):
|
||||
conn.execute('INSERT INTO job_events (job_id, event, attempt) VALUES (%s, %s, %s)', (job['id'], name, job['attempt']))
|
||||
|
||||
|
||||
def retry(conn, job, code, permanent=False):
|
||||
failed = permanent or job['attempt'] >= job['max_attempts']
|
||||
state = 'failed' if failed else 'queued'
|
||||
code = code if permanent or not failed else 'retry_exhausted'
|
||||
delay = min(8, .5 * 2 ** min(job['attempt'] - 1, 4)) + random.uniform(0, .5)
|
||||
conn.execute('''
|
||||
UPDATE attachment_jobs
|
||||
SET state = %s
|
||||
, error_code = %s
|
||||
, lease_token = NULL
|
||||
, lease_until = NULL
|
||||
, published_at = NULL
|
||||
, next_attempt_at = clock_timestamp() + %s * INTERVAL '1 second'
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
''', (state, code, delay, job['id']))
|
||||
conn.execute('''
|
||||
UPDATE attachments
|
||||
SET state = %s
|
||||
, error_code = %s
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND current_job_id = %s
|
||||
''', (state, code if failed else None, job['id']))
|
||||
audit(conn, job, 'failed:' + code if failed else 'retry:' + code)
|
||||
tele.count('job_retries', {'job_type':'image', 'outcome':'failed' if failed else 'retry'})
|
||||
tele.log('job_failed' if failed else 'job_retry', job_id=str(job['id']), attempt=job['attempt'], error_code=code)
|
||||
|
||||
|
||||
def dispatch_once(client, fault=None):
|
||||
with connect() as conn:
|
||||
with conn.transaction():
|
||||
expired = conn.execute("SELECT * FROM attachment_jobs WHERE 1=1 AND state = 'processing' AND lease_until < clock_timestamp() FOR UPDATE SKIP LOCKED LIMIT 20").fetchall()
|
||||
for job in expired:
|
||||
retry(conn, job, 'lease_expired')
|
||||
with conn.transaction():
|
||||
jobs = conn.execute('''
|
||||
SELECT j.*
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.current_job_id = j.id
|
||||
WHERE 1=1
|
||||
AND j.state = 'queued'
|
||||
AND j.next_attempt_at <= clock_timestamp()
|
||||
AND (j.published_at IS NULL OR j.published_at < clock_timestamp() - %s * INTERVAL '1 second')
|
||||
ORDER BY j.created_at
|
||||
LIMIT 10
|
||||
FOR UPDATE OF j SKIP LOCKED
|
||||
''', (LEASE,)).fetchall()
|
||||
for job in jobs:
|
||||
with tele.span('attachment publish', {'messaging.system':'redis', 'messaging.destination.name':STREAM},
|
||||
tele.SpanKind.PRODUCER, job.get('trace_context') or {}):
|
||||
message = event(job)
|
||||
message.update(tele.carrier())
|
||||
client.xadd(STREAM, {'event': json.dumps(message, separators=(',', ':'))})
|
||||
tele.log('job_published', job_id=str(job['id']), attempt=job['attempt']+1)
|
||||
if fault:
|
||||
fault('after_publish', job)
|
||||
conn.execute('UPDATE attachment_jobs SET published_at = clock_timestamp() WHERE 1=1 AND id = %s', (job['id'],))
|
||||
audit(conn, job, 'published')
|
||||
return len(jobs)
|
||||
|
||||
|
||||
def claim(message):
|
||||
with connect() as conn, conn.transaction():
|
||||
job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (message['job_id'],)).fetchone()
|
||||
if job is None:
|
||||
return None
|
||||
attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (job['attachment_id'],)).fetchone()
|
||||
if str(job['attachment_id']) != message['attachment_id'] or job['pipeline_version'] != message['pipeline_version']:
|
||||
return None
|
||||
if attachment['current_job_id'] != job['id'] or attachment['pipeline_version'] != job['pipeline_version']:
|
||||
audit(conn, job, 'stale_noop')
|
||||
return None
|
||||
if job['state'] in ('ready', 'failed', 'superseded'):
|
||||
audit(conn, job, 'terminal_noop')
|
||||
return None
|
||||
if job['state'] != 'queued' or job['next_attempt_at'] > dt.datetime.now(dt.timezone.utc) or message['attempt'] != job['attempt'] + 1:
|
||||
audit(conn, job, 'duplicate_or_early_noop')
|
||||
return None
|
||||
token = uuid.uuid4()
|
||||
job = conn.execute('''
|
||||
UPDATE attachment_jobs
|
||||
SET state = 'processing'
|
||||
, attempt = attempt + 1
|
||||
, lease_token = %s
|
||||
, lease_until = clock_timestamp() + %s * INTERVAL '1 second'
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
RETURNING *
|
||||
''', (token, LEASE, job['id'])).fetchone()
|
||||
conn.execute("UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job['id'],))
|
||||
audit(conn, job, 'claimed')
|
||||
return job, attachment
|
||||
|
||||
|
||||
class Permanent(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def immutable(objects, key, path, mime, sha):
|
||||
head = objects.head(key)
|
||||
if head is not None:
|
||||
if head['ContentLength'] != path.stat().st_size or head.get('Metadata', {}).get('sha256') != sha:
|
||||
raise Permanent('result_conflict')
|
||||
return
|
||||
try:
|
||||
with path.open('rb') as source:
|
||||
objects.client.put_object(Bucket=objects.bucket, Key=key, Body=source, ContentLength=path.stat().st_size, ContentType=mime, Metadata={'sha256': sha}, IfNoneMatch='*')
|
||||
except ClientError as error:
|
||||
if error.response['ResponseMetadata']['HTTPStatusCode'] in (409, 412):
|
||||
head = objects.head(key)
|
||||
if head and head['ContentLength'] == path.stat().st_size and head.get('Metadata', {}).get('sha256') == sha:
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
def process_core(message, fault=None):
|
||||
claimed = claim(message)
|
||||
if claimed is None:
|
||||
return 'noop'
|
||||
job, attachment = claimed
|
||||
objects = Objects(setting)
|
||||
try:
|
||||
if fault:
|
||||
fault('before_side_effect', job)
|
||||
with tempfile.TemporaryDirectory(prefix='job-') as temporary:
|
||||
directory = Path(temporary)
|
||||
source = directory / 'original'
|
||||
head = objects.head(job['source_key'])
|
||||
if head is None:
|
||||
raise Permanent('source_missing')
|
||||
response = objects.get(job['source_key'])
|
||||
digest, size = hashlib.sha256(), 0
|
||||
deadline = time.monotonic() + 15
|
||||
with response['Body'] as stream, source.open('wb') as out:
|
||||
for chunk in iter(lambda: stream.read(64 * 1024), b''):
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES or time.monotonic() > deadline:
|
||||
raise Permanent('invalid_image')
|
||||
digest.update(chunk); out.write(chunk)
|
||||
if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']:
|
||||
raise Permanent('checksum_mismatch')
|
||||
result = subprocess.run([sys.executable, str(Path(__file__).with_name('render.py')), str(source), str(directory), str(job['pipeline_version'])], capture_output=True, timeout=int(os.environ.get('JOB_TIMEOUT_SECONDS', '20')))
|
||||
if result.returncode == 2:
|
||||
raise Permanent('invalid_image')
|
||||
if result.returncode:
|
||||
raise RuntimeError('render_failed')
|
||||
variants = json.loads(result.stdout)
|
||||
prefix = 'results/' + str(attachment['id']) + '/v' + str(job['pipeline_version']) + '/'
|
||||
original_key = prefix + 'original'
|
||||
immutable(objects, original_key, source, attachment['content_type'], attachment['sha256'])
|
||||
if fault:
|
||||
fault('after_original', job)
|
||||
for variant in variants:
|
||||
variant['object_key'] = prefix + variant['name']
|
||||
immutable(objects, variant['object_key'], directory / variant['name'], variant['content_type'], variant['sha256'])
|
||||
if fault:
|
||||
fault('after_objects', job)
|
||||
with connect() as conn, conn.transaction():
|
||||
current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone()
|
||||
changed = conn.execute('''
|
||||
UPDATE attachments
|
||||
SET state = 'ready'
|
||||
, object_key = %s
|
||||
, variants = %s
|
||||
, error_code = NULL
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND current_job_id = %s
|
||||
AND pipeline_version = %s
|
||||
AND %s
|
||||
RETURNING id
|
||||
''', (original_key, Jsonb(variants), job['id'], job['pipeline_version'], current['state'] == 'processing' and current['lease_token'] == job['lease_token'] and current['lease_until'] > dt.datetime.now(dt.timezone.utc))).fetchone()
|
||||
if changed:
|
||||
conn.execute("UPDATE attachment_jobs SET state = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE 1=1 AND id = %s", (job['id'],))
|
||||
audit(conn, job, 'ready')
|
||||
else:
|
||||
audit(conn, job, 'fenced_noop')
|
||||
if fault:
|
||||
fault('after_commit', job)
|
||||
if changed and job['source_key'].startswith('staging/'):
|
||||
objects.delete(job['source_key'])
|
||||
if changed:
|
||||
tele.duration('job_ready_duration', (dt.datetime.now(dt.timezone.utc)-job['created_at']).total_seconds(), {'job_type':'image'})
|
||||
return 'ready' if changed else 'fenced'
|
||||
except Exception as error:
|
||||
code = str(error) if isinstance(error, Permanent) else 'processing_timeout' if isinstance(error, subprocess.TimeoutExpired) else 'storage_unavailable' if isinstance(error, (APIError, ClientError, BotoCoreError)) else 'processing_failed'
|
||||
with connect() as conn, conn.transaction():
|
||||
current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone()
|
||||
if current['state'] == 'processing' and current['lease_token'] == job['lease_token']:
|
||||
retry(conn, current, code, isinstance(error, Permanent))
|
||||
return 'failed_or_retry'
|
||||
finally:
|
||||
objects.client.close()
|
||||
|
||||
|
||||
def process(message, fault=None):
|
||||
started = time.monotonic()
|
||||
with tele.span('attachment process', {'messaging.system':'redis', 'messaging.destination.name':STREAM,
|
||||
'job.id':message['job_id'], 'job.attempt':message['attempt']},
|
||||
tele.SpanKind.CONSUMER, message):
|
||||
with tele.LOCK:
|
||||
tele.STATE['worker_busy'] += 1
|
||||
outcome = 'error'
|
||||
try:
|
||||
outcome = process_core(message, fault)
|
||||
if outcome == 'failed_or_retry':
|
||||
tele.trace.get_current_span().set_status(tele.Status(tele.StatusCode.ERROR))
|
||||
return outcome
|
||||
finally:
|
||||
with tele.LOCK:
|
||||
tele.STATE['worker_busy'] -= 1
|
||||
attributes = {'job_type':'image', 'outcome':outcome}
|
||||
tele.count('jobs_processed', attributes)
|
||||
tele.duration('job_processing_duration', time.monotonic()-started, attributes)
|
||||
tele.log('job_processed', job_id=message['job_id'], attempt=message['attempt'], outcome=outcome)
|
||||
|
||||
|
||||
def observe_queue():
|
||||
try:
|
||||
with raw_connect() as conn:
|
||||
row = conn.execute("""
|
||||
SELECT
|
||||
count(*) AS depth
|
||||
, coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS age
|
||||
FROM attachment_jobs
|
||||
WHERE 1=1
|
||||
AND state IN ('queued', 'processing')
|
||||
""").fetchone()
|
||||
tele.STATE.update(queue_depth=row['depth'], queue_oldest_age=float(row['age']), queue_observation_ok=1)
|
||||
except psycopg.Error:
|
||||
tele.STATE['queue_observation_ok'] = 0
|
||||
|
||||
|
||||
def ack(client, identifier):
|
||||
# Atomic ack+delete; PostgreSQL is the durable completion/replay registry.
|
||||
client.eval("redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])", 1, STREAM, GROUP, identifier)
|
||||
|
||||
|
||||
def consume(once=False, fault=None):
|
||||
client = broker()
|
||||
consumer = uuid.uuid4().hex
|
||||
while not STOP.is_set():
|
||||
messages = []
|
||||
try:
|
||||
pending = client.xautoclaim(STREAM, GROUP, consumer, min_idle_time=LEASE * 1000, start_id='0-0', count=1)[1]
|
||||
if pending:
|
||||
messages = pending
|
||||
else:
|
||||
batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=500)
|
||||
messages = batches[0][1] if batches else []
|
||||
for identifier, fields in messages:
|
||||
raw = fields.get('event', '')
|
||||
try:
|
||||
message = json.loads(raw) if len(raw) <= 4096 else None
|
||||
except ValueError:
|
||||
message = None
|
||||
if validate(message):
|
||||
process(message, fault)
|
||||
else:
|
||||
client.xadd('rejected-jobs', {'source_message_id': identifier, 'sha256': hashlib.sha256(raw.encode()).hexdigest(), 'reason': 'invalid_envelope'}, maxlen=1000, approximate=False)
|
||||
ack(client, identifier)
|
||||
if once:
|
||||
return
|
||||
except (redis.RedisError, psycopg.Error):
|
||||
STOP.wait(.5)
|
||||
if once and not messages:
|
||||
return
|
||||
client.close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('mode', choices=['dispatch', 'work'])
|
||||
parser.add_argument('--once', action='store_true')
|
||||
parser.add_argument('--fault-point', choices=['before_side_effect', 'after_original', 'after_objects', 'after_commit', 'after_publish'])
|
||||
args = parser.parse_args()
|
||||
tele.configure('messenger-dispatcher' if args.mode=='dispatch' else 'messenger-worker')
|
||||
# Operator-only opt-in injection; no request header or public endpoint controls it.
|
||||
def fault(point, job):
|
||||
if point == args.fault_point:
|
||||
os._exit(77)
|
||||
signal.signal(signal.SIGTERM, lambda *_: STOP.set())
|
||||
signal.signal(signal.SIGINT, lambda *_: STOP.set())
|
||||
if args.mode == 'dispatch':
|
||||
client = broker()
|
||||
while not STOP.is_set():
|
||||
observe_queue()
|
||||
try:
|
||||
dispatch_once(client, fault)
|
||||
except (redis.RedisError, psycopg.Error):
|
||||
pass
|
||||
if args.once:
|
||||
break
|
||||
STOP.wait(.2)
|
||||
else:
|
||||
count = int(os.environ.get('WORKER_CONCURRENCY', '1'))
|
||||
if not 1 <= count <= 2:
|
||||
raise ValueError('WORKER_CONCURRENCY must be 1..2')
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool:
|
||||
futures = [pool.submit(consume, args.once, fault) for _ in range(count)]
|
||||
for result in futures:
|
||||
result.result()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
finally:
|
||||
tele.shutdown()
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
import base64
|
||||
import contextlib
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import threading
|
||||
import uuid
|
||||
import telemetry as tele
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
import redis
|
||||
|
||||
from objects import Objects, unavailable
|
||||
|
||||
from common import APIError, fields, identifier, invalid, missing, page, text, unauthorized, username
|
||||
|
||||
KDF_SLOTS = threading.BoundedSemaphore(2)
|
||||
|
||||
|
||||
def setting(name):
|
||||
path = os.environ.get(name + '_FILE')
|
||||
return Path(path).read_text().strip() if path else os.environ[name]
|
||||
|
||||
|
||||
def public(row):
|
||||
return {k: (v.decode('utf-8') if isinstance(v, bytes) else [str(x) for x in v] if isinstance(v, list) else str(v) if isinstance(v, uuid.UUID) else v.isoformat(timespec='microseconds').replace('+00:00', 'Z') if isinstance(v, dt.datetime) else v) for k, v in row.items() if k != 'password_hash'}
|
||||
|
||||
|
||||
def password_hash(password, salt=None):
|
||||
salt = salt or secrets.token_bytes(16)
|
||||
if not KDF_SLOTS.acquire(timeout=1.5):
|
||||
raise APIError(503, 'unavailable', 'Аутентификация временно перегружена')
|
||||
try:
|
||||
digest = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024)
|
||||
finally:
|
||||
KDF_SLOTS.release()
|
||||
return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex()
|
||||
|
||||
|
||||
class Store:
|
||||
persistent = True
|
||||
|
||||
def __init__(self):
|
||||
self.ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600'))
|
||||
if not 1 <= self.ttl <= 86400:
|
||||
raise ValueError('SESSION_TTL_SECONDS must be 1..86400')
|
||||
self.access_mode = os.environ.get('DB_ACCESS_MODE', 'optimized')
|
||||
if self.access_mode not in ('baseline', 'optimized'):
|
||||
raise ValueError('DB_ACCESS_MODE must be baseline or optimized')
|
||||
pool_size = int(os.environ.get('DB_POOL_MAX', '8'))
|
||||
if not 1 <= pool_size <= 32:
|
||||
raise ValueError('DB_POOL_MAX must be 1..32')
|
||||
self.pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=pool_size, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=2000 -c lock_timeout=1500'}, open=True)
|
||||
self.pool.wait(timeout=10)
|
||||
tele.pool(self.pool)
|
||||
self.cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.15, socket_timeout=0.15, decode_responses=True)
|
||||
self.objects = Objects(setting)
|
||||
self.dummy_hash = password_hash('dummy-password-for-unknown-users')
|
||||
with self.connection() as conn:
|
||||
if not conn.execute("SELECT version FROM schema_migrations WHERE 1=1 AND version = '006_observability.sql'").fetchone():
|
||||
raise RuntimeError('Run the separate migration service first')
|
||||
|
||||
@contextlib.contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
with tele.span('db.transaction', {'db.system':'postgresql'}, tele.SpanKind.CLIENT):
|
||||
with self.pool.connection() as conn:
|
||||
yield conn
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise APIError(409, 'conflict', 'Имя пользователя занято') from None
|
||||
except (psycopg.Error, PoolTimeout):
|
||||
raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None
|
||||
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
self.cache.close()
|
||||
self.objects.client.close()
|
||||
|
||||
def ready(self):
|
||||
self.objects.ready()
|
||||
with self.connection() as conn:
|
||||
conn.execute('SELECT 1')
|
||||
|
||||
def register(self, body):
|
||||
fields(body, ('username', 'display_name', 'password'))
|
||||
name, display = username(body['username']), text(body['display_name'], 100)
|
||||
# Passwords may contain spaces and colons; only length/UTF-8 are restricted.
|
||||
password = body['password']
|
||||
if not isinstance(password, str) or not 12 <= len(password) <= 128:
|
||||
invalid()
|
||||
try:
|
||||
password.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
hashed = password_hash(password)
|
||||
with self.connection() as conn:
|
||||
row = conn.execute('''
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
RETURNING id, username, display_name, created_at
|
||||
''', (uuid.uuid4(), name, display.encode('utf-8'), hashed)).fetchone()
|
||||
return public(row)
|
||||
|
||||
def login(self, headers):
|
||||
authorization = headers.get('Authorization', '')
|
||||
try:
|
||||
scheme, encoded = authorization.split(' ', 1)
|
||||
if scheme.lower() != 'basic':
|
||||
unauthorized()
|
||||
name, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
except (ValueError, UnicodeError):
|
||||
unauthorized()
|
||||
if not re_username(name) or not 12 <= len(password) <= 128:
|
||||
unauthorized()
|
||||
with self.connection() as conn:
|
||||
row = conn.execute('SELECT id, username, display_name, created_at, password_hash FROM users WHERE 1=1 AND username = %s', (name,)).fetchone()
|
||||
hashed = row['password_hash'] if row else self.dummy_hash
|
||||
verified = hmac.compare_digest(password_hash(password, bytes.fromhex(hashed.split('$')[4])), hashed)
|
||||
if row is None or not verified:
|
||||
unauthorized()
|
||||
token = secrets.token_urlsafe(32)
|
||||
with self.connection() as conn:
|
||||
expiry = conn.execute('''
|
||||
INSERT INTO sessions (token_hash, user_id, expires_at)
|
||||
VALUES (%s, %s, clock_timestamp() + %s * INTERVAL '1 second')
|
||||
RETURNING expires_at
|
||||
''', (hashlib.sha256(token.encode()).hexdigest(), row['id'], self.ttl)).fetchone()['expires_at']
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': public({'expires_at': expiry})['expires_at'], 'user': public(row)}
|
||||
|
||||
def token_hash(self, headers):
|
||||
parts = headers.get('Authorization', '').split(' ')
|
||||
if len(parts) != 2 or parts[0].lower() != 'bearer' or len(parts[1]) != 43 or not parts[1].isascii():
|
||||
unauthorized()
|
||||
return hashlib.sha256(parts[1].encode()).hexdigest()
|
||||
|
||||
def authenticate(self, headers):
|
||||
token_hash = self.token_hash(headers)
|
||||
with self.connection() as conn:
|
||||
session = conn.execute('''
|
||||
SELECT user_id
|
||||
FROM sessions
|
||||
WHERE 1=1
|
||||
AND token_hash = %s
|
||||
AND expires_at > clock_timestamp()
|
||||
''', (token_hash,)).fetchone()
|
||||
if session is None:
|
||||
unauthorized()
|
||||
key = 'user:' + str(session['user_id'])
|
||||
try:
|
||||
cached = self.cache.get(key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (redis.RedisError, ValueError):
|
||||
pass
|
||||
row = conn.execute('SELECT id, username, display_name, created_at FROM users WHERE 1=1 AND id = %s', (session['user_id'],)).fetchone()
|
||||
if row is None:
|
||||
unauthorized()
|
||||
user = public(row)
|
||||
try:
|
||||
self.cache.setex(key, 60, json.dumps(user))
|
||||
except redis.RedisError:
|
||||
pass
|
||||
return user
|
||||
|
||||
def logout(self, headers):
|
||||
with self.connection() as conn:
|
||||
conn.execute('DELETE FROM sessions WHERE 1=1 AND token_hash = %s', (self.token_hash(headers),))
|
||||
|
||||
def chat_in(self, conn, actor, chat_id):
|
||||
row = conn.execute('''
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE 1=1
|
||||
AND c.id = %s
|
||||
AND cm.user_id = %s
|
||||
''', (chat_id, actor['id'])).fetchone()
|
||||
if row is None:
|
||||
missing()
|
||||
members = conn.execute('SELECT user_id FROM chat_members WHERE 1=1 AND chat_id = %s ORDER BY user_id', (chat_id,)).fetchall()
|
||||
return dict(public(row), member_ids=[str(v['user_id']) for v in members])
|
||||
|
||||
def chat(self, actor, chat_id):
|
||||
with self.connection() as conn:
|
||||
return self.chat_in(conn, actor, chat_id)
|
||||
|
||||
def create_chat(self, actor, body):
|
||||
fields(body, ('title', 'member_ids'))
|
||||
title = text(body['title'], 100)
|
||||
members = body['member_ids']
|
||||
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||
invalid()
|
||||
members = [identifier(v) for v in members]
|
||||
if len(set(members)) != len(members) or actor['id'] not in members:
|
||||
invalid()
|
||||
with self.connection() as conn:
|
||||
chat_id = uuid.uuid4()
|
||||
conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat_id, title.encode('utf-8')))
|
||||
# Checking after INSERT exercises rollback of the whole composite operation.
|
||||
found = conn.execute('SELECT id FROM users WHERE 1=1 AND id = ANY(%s::uuid[])', (members,)).fetchall()
|
||||
if len(found) != len(members):
|
||||
invalid()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', [(chat_id, v) for v in members])
|
||||
return self.chat_in(conn, actor, chat_id)
|
||||
|
||||
def list_chats(self, actor, limit, cursor, scope):
|
||||
with self.connection() as conn:
|
||||
where = 'AND (c.created_at, c.id) > (%s::timestamptz, %s::uuid)' if cursor else ''
|
||||
params = [actor['id']] + list(cursor or ()) + [limit + 1]
|
||||
rows = conn.execute('''
|
||||
SELECT c.id
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE 1=1
|
||||
AND cm.user_id = %s
|
||||
''' + where + '''
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
''', params).fetchall()
|
||||
return page([self.chat_in(conn, actor, v['id']) for v in rows], limit, scope)
|
||||
|
||||
def require_member(self, conn, actor, chat_id):
|
||||
if self.access_mode == 'baseline':
|
||||
return self.chat_in(conn, actor, chat_id)
|
||||
if not conn.execute('''
|
||||
SELECT 1
|
||||
FROM chat_members
|
||||
WHERE 1=1
|
||||
AND chat_id = %s
|
||||
AND user_id = %s
|
||||
''', (chat_id, actor['id'])).fetchone():
|
||||
missing()
|
||||
|
||||
def send(self, actor, chat_id, body, key):
|
||||
if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
|
||||
invalid()
|
||||
if not isinstance(body, dict) or set(body) - {'text', 'attachment_ids'} or 'text' not in body:
|
||||
invalid()
|
||||
attachments = body.get('attachment_ids', [])
|
||||
if not isinstance(attachments, list) or len(attachments) > 10:
|
||||
invalid()
|
||||
attachments = [identifier(v) for v in attachments]
|
||||
if len(set(attachments)) != len(attachments):
|
||||
invalid()
|
||||
content = body['text']
|
||||
if not isinstance(content, str) or len(content) > 4000 or (not attachments and not content.strip()):
|
||||
invalid()
|
||||
try:
|
||||
content.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
canonical = json.dumps({'text': content, 'attachment_ids': attachments}, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
||||
digest = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
|
||||
with self.connection() as conn:
|
||||
self.require_member(conn, actor, chat_id)
|
||||
# The unique key arbitrates concurrent processes; everything commits together.
|
||||
claimed = conn.execute('''
|
||||
INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, expires_at)
|
||||
VALUES (%s, %s, %s, %s, clock_timestamp() + INTERVAL '24 hours')
|
||||
ON CONFLICT (user_id, chat_id, key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash
|
||||
, message_id = NULL
|
||||
, expires_at = EXCLUDED.expires_at
|
||||
WHERE 1=1
|
||||
AND idempotency_keys.expires_at <= clock_timestamp()
|
||||
RETURNING request_hash
|
||||
''', (actor['id'], chat_id, key, digest)).fetchone()
|
||||
if claimed is None:
|
||||
previous = conn.execute('''
|
||||
SELECT ik.request_hash
|
||||
, m.id
|
||||
, m.chat_id
|
||||
, m.sender_id
|
||||
, m.text
|
||||
, m.created_at
|
||||
, m.attachment_ids
|
||||
FROM idempotency_keys ik
|
||||
JOIN messages m ON m.id = ik.message_id
|
||||
WHERE 1=1
|
||||
AND ik.user_id = %s
|
||||
AND ik.chat_id = %s
|
||||
AND ik.key = %s
|
||||
''', (actor['id'], chat_id, key)).fetchone()
|
||||
if previous['request_hash'] != digest:
|
||||
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
|
||||
previous.pop('request_hash')
|
||||
return public(previous)
|
||||
if attachments:
|
||||
rows = conn.execute('''
|
||||
SELECT id, state
|
||||
FROM attachments
|
||||
WHERE 1=1
|
||||
AND id = ANY(%s::uuid[])
|
||||
AND chat_id = %s
|
||||
ORDER BY id
|
||||
FOR SHARE
|
||||
''', (attachments, chat_id)).fetchall()
|
||||
if len(rows) != len(attachments):
|
||||
missing()
|
||||
if any(v['state'] not in ('ready', 'queued', 'processing') for v in rows):
|
||||
raise APIError(409, 'conflict', 'Вложение ещё не готово')
|
||||
# Per-chat row lock preserves committed history order across replicas/clocks.
|
||||
stamp = conn.execute('''
|
||||
UPDATE chats
|
||||
SET last_message_at = GREATEST(clock_timestamp(), last_message_at + INTERVAL '1 microsecond')
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
RETURNING last_message_at
|
||||
''', (chat_id,)).fetchone()['last_message_at']
|
||||
row = conn.execute('''
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
|
||||
VALUES (%s, %s, %s, %s, %s, %s::uuid[])
|
||||
RETURNING id, chat_id, sender_id, text, created_at, attachment_ids
|
||||
''', (uuid.uuid4(), chat_id, actor['id'], content.encode('utf-8'), stamp, attachments)).fetchone()
|
||||
if attachments:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)', [(row['id'], v) for v in attachments])
|
||||
conn.execute('''
|
||||
UPDATE idempotency_keys
|
||||
SET message_id = %s
|
||||
WHERE 1=1
|
||||
AND user_id = %s
|
||||
AND chat_id = %s
|
||||
AND key = %s
|
||||
''', (row['id'], actor['id'], chat_id, key))
|
||||
return public(row)
|
||||
|
||||
def history(self, actor, chat_id, limit, cursor, scope):
|
||||
with self.connection() as conn:
|
||||
self.require_member(conn, actor, chat_id)
|
||||
where = 'AND (created_at, id) > (%s::timestamptz, %s::uuid)' if cursor else ''
|
||||
params = [chat_id] + list(cursor or ()) + [limit + 1]
|
||||
rows = conn.execute('''
|
||||
SELECT id, chat_id, sender_id, text, created_at, attachment_ids
|
||||
FROM messages
|
||||
WHERE 1=1
|
||||
AND chat_id = %s
|
||||
''' + where + '''
|
||||
ORDER BY created_at, id
|
||||
LIMIT %s
|
||||
''', params).fetchall()
|
||||
return page([public(v) for v in rows], limit, scope)
|
||||
|
||||
def upload(self, actor, chat_id, name, source, length, mime, digest):
|
||||
attachment_id, job_id = uuid.uuid4(), uuid.uuid4()
|
||||
key = 'staging/' + str(attachment_id)
|
||||
with self.connection() as conn:
|
||||
self.require_member(conn, actor, chat_id)
|
||||
conn.execute('SELECT pg_advisory_xact_lock(2301005)')
|
||||
pending = conn.execute("SELECT count(*) AS n FROM attachments WHERE 1=1 AND state IN ('uploading', 'queued', 'processing')").fetchone()['n']
|
||||
if pending >= int(os.environ.get('MAX_PENDING_JOBS', '100')):
|
||||
raise APIError(503, 'unavailable', 'Очередь принятых работ заполнена')
|
||||
conn.execute('''
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256, state)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')
|
||||
''', (attachment_id, chat_id, actor['id'], name, 'results/' + str(attachment_id) + '/v1/original', key, mime, length, digest))
|
||||
with self.connection() as conn:
|
||||
if conn.execute('SELECT id FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() is None:
|
||||
unavailable()
|
||||
self.objects.put(key, source, length, mime, digest)
|
||||
conn.execute('''
|
||||
INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts, trace_context)
|
||||
VALUES (%s, %s, 1, %s, %s, %s)
|
||||
''', (job_id, attachment_id, key, int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), Jsonb(tele.carrier())))
|
||||
row = conn.execute('''
|
||||
UPDATE attachments
|
||||
SET state = 'queued'
|
||||
, current_job_id = %s
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
RETURNING *
|
||||
''', (job_id, attachment_id)).fetchone()
|
||||
tele.log('attachment_accepted', attachment_id=str(attachment_id), job_id=str(job_id))
|
||||
# Capture the queued snapshot, even if the worker finishes before HTTP reply.
|
||||
return self.attachment_public(row)
|
||||
|
||||
def attachment_public(self, row):
|
||||
keys = ('id', 'chat_id', 'owner_id', 'filename', 'content_type', 'size_bytes', 'sha256', 'created_at', 'updated_at')
|
||||
result = public({k: row[k] for k in keys})
|
||||
variants = [{k: v for k, v in item.items() if k != 'object_key'} for item in row['variants']]
|
||||
return dict(result, status=row['state'], variants=variants, error_code=row['error_code'])
|
||||
|
||||
def attachment(self, actor, attachment_id):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute('''
|
||||
SELECT a.*
|
||||
FROM attachments a
|
||||
JOIN chat_members cm ON cm.chat_id = a.chat_id
|
||||
WHERE 1=1
|
||||
AND a.id = %s
|
||||
AND cm.user_id = %s
|
||||
''', (attachment_id, actor['id'])).fetchone()
|
||||
if row is None:
|
||||
missing()
|
||||
if row['state'] == 'storage_missing':
|
||||
unavailable()
|
||||
if row['state'] == 'uploading':
|
||||
raise APIError(409, 'conflict', 'Приём не завершён')
|
||||
return row
|
||||
|
||||
|
||||
def re_username(value):
|
||||
import re
|
||||
return re.fullmatch(r'[a-z0-9_]{3,32}', value)
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Isolated image decoder/resizer; no network, database or credentials."""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (15, 16))
|
||||
from PIL import Image, ImageOps, PngImagePlugin
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
PngImagePlugin.MAX_TEXT_CHUNK = PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
|
||||
warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
def render(source, directory, version):
|
||||
output = []
|
||||
with Image.open(source, formats=['PNG', 'JPEG']) as image:
|
||||
if image.width * image.height > 40_000_000:
|
||||
raise ValueError('pixels')
|
||||
image.load()
|
||||
ImageOps.exif_transpose(image, in_place=True)
|
||||
rgba = None
|
||||
if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info):
|
||||
rgba = image if image.mode == 'RGBA' else image.convert('RGBA')
|
||||
rgb = Image.new('RGB', rgba.size, 'white')
|
||||
alpha = rgba.getchannel('A')
|
||||
rgb.paste(rgba, mask=alpha)
|
||||
alpha.close()
|
||||
else:
|
||||
rgb = image if image.mode == 'RGB' else image.convert('RGB')
|
||||
try:
|
||||
for name, bound, fmt, quality in [('thumbnail', 256, 'JPEG', 85), ('crop', 128, 'JPEG', 85), ('optimized', 1280, 'WEBP', 82 if version == 1 else 70)]:
|
||||
if name == 'crop':
|
||||
result = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(.5, .5))
|
||||
else:
|
||||
scale = min(1, bound / max(rgb.size))
|
||||
size = tuple(max(1, math.floor(v * scale)) for v in rgb.size)
|
||||
result = rgb.resize(size, Image.Resampling.LANCZOS)
|
||||
try:
|
||||
path = directory / name
|
||||
options = {'quality': quality, 'exif': b'', 'icc_profile': None}
|
||||
if fmt == 'WEBP':
|
||||
options['method'] = 4
|
||||
result.save(path, format=fmt, **options)
|
||||
raw = path.read_bytes()
|
||||
output.append({'name': name, 'width': result.width, 'height': result.height, 'content_type': 'image/jpeg' if fmt == 'JPEG' else 'image/webp', 'size_bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()})
|
||||
finally:
|
||||
result.close()
|
||||
finally:
|
||||
if rgb is not image:
|
||||
rgb.close()
|
||||
if rgba is not None and rgba is not image:
|
||||
rgba.close()
|
||||
return output
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
print(json.dumps(render(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3]))))
|
||||
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise SystemExit(2)
|
||||
+317
@@ -0,0 +1,317 @@
|
||||
"""Bounded threaded API; persistent correctness state is shared across replicas."""
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import http.server
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import signal
|
||||
import socket
|
||||
import threading
|
||||
import uuid
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from objects import bounded_upload, receive, filename, unavailable
|
||||
|
||||
import telemetry as tele
|
||||
|
||||
from common import APIError, identifier, invalid, missing, page_args
|
||||
|
||||
LOG = logging.getLogger('messenger')
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
invalid()
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.1'
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.connection.settimeout(3)
|
||||
|
||||
def send_error(self, code, message=None, explain=None):
|
||||
# BaseHTTPRequestHandler otherwise emits HTML without contract headers.
|
||||
self.request_id = uuid.uuid4().hex
|
||||
self.route = ''
|
||||
self.close_connection = True
|
||||
status = 413 if code == 413 else 400
|
||||
self.reply(status, {'error': {'code': 'payload_too_large' if status == 413 else 'invalid_request', 'message': 'Некорректный HTTP-запрос', 'request_id': self.request_id}})
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def reply(self, status, body):
|
||||
if status == 200 and isinstance(body, tuple):
|
||||
return self.download(*body)
|
||||
raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8')
|
||||
if status >= 400:
|
||||
self.close_connection = True
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.send_header('Content-Length', str(len(raw)))
|
||||
if self.close_connection:
|
||||
self.send_header('Connection', 'close')
|
||||
self.send_header('X-Request-Id', self.request_id)
|
||||
self.send_header('X-Instance-Id', self.server.instance_id)
|
||||
if status == 401 and self.server.store.persistent:
|
||||
self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
|
||||
self.end_headers()
|
||||
if self.command != 'HEAD':
|
||||
self.wfile.write(raw)
|
||||
|
||||
def download(self, row, response, chunk):
|
||||
stream = response['Body']
|
||||
try:
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', row['content_type'])
|
||||
self.send_header('Content-Length', str(row['size_bytes']))
|
||||
self.send_header('X-Request-Id', self.request_id)
|
||||
self.send_header('X-Instance-Id', self.server.instance_id)
|
||||
self.send_header('X-Content-Type-Options', 'nosniff')
|
||||
self.send_header('Cache-Control', 'private, no-store')
|
||||
self.end_headers()
|
||||
deadline = time.monotonic() + 15
|
||||
while chunk:
|
||||
self.wfile.write(chunk)
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError()
|
||||
chunk = stream.read(64 * 1024)
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
def body(self):
|
||||
lengths = self.headers.get_all('Content-Length', [])
|
||||
if len(lengths) != 1 or self.headers.get('Transfer-Encoding'):
|
||||
invalid()
|
||||
try:
|
||||
length = int(lengths[0])
|
||||
except ValueError:
|
||||
invalid()
|
||||
if length > 32768:
|
||||
raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт')
|
||||
if length <= 0:
|
||||
invalid()
|
||||
if self.headers.get_content_type() != 'application/json':
|
||||
raise APIError(415, 'unsupported_media_type', 'Требуется application/json')
|
||||
try:
|
||||
raw = self.rfile.read(length)
|
||||
if len(raw) != length:
|
||||
invalid()
|
||||
return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid())
|
||||
except (ValueError, UnicodeError, RecursionError):
|
||||
invalid()
|
||||
|
||||
def dispatch(self):
|
||||
self.request_id = uuid.uuid4().hex
|
||||
self.route = urlsplit(self.path).path
|
||||
template = tele.route(self.route)
|
||||
method = self.command if self.command in ('GET','POST','DELETE','PUT','PATCH','HEAD','OPTIONS') else 'OTHER'
|
||||
attrs = {'method':method, 'route':template}
|
||||
started = time.monotonic()
|
||||
context = tele.span(method+' '+template, {'http.request.method':method, 'http.route':template},
|
||||
tele.SpanKind.SERVER, {k:self.headers[k] for k in ('traceparent','tracestate') if k in self.headers})
|
||||
# Health probes do not enter business telemetry or create noisy dependency spans.
|
||||
if template.startswith('/health/'):
|
||||
context = tele.suppressed()
|
||||
status = 500
|
||||
with context as current:
|
||||
try:
|
||||
status, result = self.route_request()
|
||||
except APIError as exc:
|
||||
status = exc.status
|
||||
result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}}
|
||||
except Exception:
|
||||
result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}}
|
||||
try:
|
||||
self.reply(status, result)
|
||||
except (BrokenPipeError, ConnectionResetError, TimeoutError, OSError):
|
||||
self.close_connection = True
|
||||
if not template.startswith('/health/'):
|
||||
attrs['status_class'] = str(status//100)+'xx'
|
||||
tele.count('http_server_requests', attrs)
|
||||
tele.duration('http_server_duration', time.monotonic()-started, attrs)
|
||||
if current is not None:
|
||||
current.set_attribute('http.response.status_code', status)
|
||||
if status >= 500:
|
||||
current.set_status(tele.Status(tele.StatusCode.ERROR))
|
||||
tele.log('http_request', request_id=self.request_id, method=method, route=template, status=status, duration_seconds=round(time.monotonic()-started,6))
|
||||
|
||||
def route_request(self):
|
||||
method, path, store = self.command, self.route, self.server.store
|
||||
if method == 'GET' and path in ('/health/live', '/health/ready'):
|
||||
if path.endswith('ready'):
|
||||
store.ready()
|
||||
return 200, {'status': 'ok'}
|
||||
if method == 'POST' and path == '/api/v1/users':
|
||||
return 201, store.register(self.body())
|
||||
if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions':
|
||||
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
|
||||
invalid()
|
||||
return 201, store.login(self.headers)
|
||||
actor = store.authenticate(self.headers)
|
||||
if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current':
|
||||
store.logout(self.headers)
|
||||
return 204, None
|
||||
if method == 'GET' and path == '/api/v1/users/me':
|
||||
return 200, actor
|
||||
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
|
||||
if path == '/api/v1/chats':
|
||||
if method == 'POST':
|
||||
return 201, store.create_chat(actor, self.body())
|
||||
if method == 'GET':
|
||||
scope = 'chats:' + actor['id']
|
||||
limit, cursor = page_args(query, scope)
|
||||
return 200, store.list_chats(actor, limit, cursor, scope)
|
||||
parts = path.split('/')
|
||||
if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'attachments'] and method == 'GET':
|
||||
row = store.attachment(actor, identifier(parts[4]))
|
||||
if len(parts) == 5:
|
||||
if query:
|
||||
invalid()
|
||||
return 200, store.attachment_public(row)
|
||||
if parts[5] == 'content':
|
||||
if set(query) - {'variant'} or any(len(v) != 1 for v in query.values()):
|
||||
invalid()
|
||||
if row['state'] != 'ready':
|
||||
raise APIError(409, 'conflict', 'Вложение ещё не готово')
|
||||
variant = query.get('variant', ['original'])[0]
|
||||
if variant != 'original':
|
||||
item = next((v for v in row['variants'] if v['name'] == variant), None)
|
||||
if item is None:
|
||||
missing()
|
||||
row = dict(row, object_key=item['object_key'], content_type=item['content_type'], size_bytes=item['size_bytes'])
|
||||
response = store.objects.get(row['object_key'])
|
||||
try:
|
||||
chunk = response['Body'].read(64 * 1024)
|
||||
if response['ContentLength'] != row['size_bytes'] or not chunk:
|
||||
unavailable()
|
||||
except Exception:
|
||||
response['Body'].close()
|
||||
unavailable()
|
||||
return 200, (row, response, chunk)
|
||||
if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']:
|
||||
chat_id = identifier(parts[4])
|
||||
if store.access_mode == 'baseline':
|
||||
store.chat(actor, chat_id)
|
||||
if len(parts) == 5 and method == 'GET':
|
||||
return 200, store.chat(actor, chat_id)
|
||||
if len(parts) == 6 and parts[5] == 'attachments' and method == 'POST':
|
||||
store.chat(actor, chat_id)
|
||||
if set(query) != {'filename'} or len(query['filename']) != 1:
|
||||
invalid()
|
||||
name = filename(query['filename'][0])
|
||||
with bounded_upload():
|
||||
with receive(self) as (source, length, mime, digest):
|
||||
return 202, store.upload(actor, chat_id, name, source, length, mime, digest)
|
||||
if len(parts) == 6 and parts[5] == 'messages':
|
||||
if method == 'POST':
|
||||
keys = self.headers.get_all('Idempotency-Key', [])
|
||||
return 201, store.send(actor, chat_id, self.body(), keys[0] if len(keys) == 1 else None)
|
||||
if method == 'GET':
|
||||
scope = 'messages:' + chat_id
|
||||
limit, cursor = page_args(query, scope)
|
||||
return 200, store.history(actor, chat_id, limit, cursor, scope)
|
||||
missing()
|
||||
|
||||
do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch
|
||||
|
||||
|
||||
class Server(http.server.HTTPServer):
|
||||
request_queue_size = 64
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, address, store):
|
||||
self.store = store
|
||||
self.instance_id = tele.INSTANCE
|
||||
workers = int(os.environ.get('HTTP_WORKERS', '16'))
|
||||
if not 1 <= workers <= 128:
|
||||
raise ValueError('HTTP_WORKERS must be 1..128')
|
||||
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers)
|
||||
self.slots = threading.BoundedSemaphore(workers * 2)
|
||||
self.connections = set()
|
||||
self.connections_lock = threading.Lock()
|
||||
super().__init__(address, Handler)
|
||||
|
||||
def process_request(self, request, address):
|
||||
if not self.slots.acquire(blocking=False):
|
||||
rid = uuid.uuid4().hex
|
||||
raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode()
|
||||
try:
|
||||
request.settimeout(0.2)
|
||||
request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Instance-Id: ' + self.instance_id + '\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw)
|
||||
except OSError:
|
||||
pass
|
||||
self.shutdown_request(request)
|
||||
return
|
||||
with self.connections_lock:
|
||||
self.connections.add(request)
|
||||
self.executor.submit(self.worker, request, address)
|
||||
|
||||
def worker(self, request, address):
|
||||
try:
|
||||
self.finish_request(request, address)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
self.shutdown_request(request)
|
||||
with self.connections_lock:
|
||||
self.connections.discard(request)
|
||||
self.slots.release()
|
||||
|
||||
def close_connections(self):
|
||||
with self.connections_lock:
|
||||
connections = list(self.connections)
|
||||
for connection in connections:
|
||||
try:
|
||||
connection.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def server_close(self):
|
||||
super().server_close()
|
||||
# Bound draining even when a slow client keeps an incomplete request alive.
|
||||
deadline = threading.Timer(5, self.close_connections)
|
||||
deadline.start()
|
||||
try:
|
||||
self.executor.shutdown(wait=True)
|
||||
finally:
|
||||
deadline.cancel()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s')
|
||||
tele.configure('messenger-api')
|
||||
from postgres import Store
|
||||
store = Store()
|
||||
server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store)
|
||||
stopping = threading.Event()
|
||||
|
||||
def stop(signum, frame):
|
||||
if not stopping.is_set():
|
||||
stopping.set()
|
||||
LOG.info('shutdown_started signal=%s', signum)
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGUSR1, lambda *_: LOG.warning('pool_stats instance=%s stats=%s', server.instance_id, json.dumps(store.pool.get_stats())))
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
LOG.info('started port=%s', server.server_address[1])
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.1)
|
||||
finally:
|
||||
server.server_close()
|
||||
store.close()
|
||||
tele.shutdown()
|
||||
LOG.info('shutdown_complete')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Allowlisted manual SDK telemetry; bounded exporters never carry business payloads."""
|
||||
import contextlib
|
||||
import contextvars
|
||||
import datetime as dt
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import resource
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from opentelemetry import trace, metrics
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider, SpanProcessor
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult
|
||||
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
|
||||
INSTANCE = uuid.uuid4().hex
|
||||
ACTIVE = False
|
||||
SUPPRESSED = contextvars.ContextVar('telemetry_suppressed', default=False)
|
||||
PROVIDERS = []
|
||||
COUNTERS = {}
|
||||
HISTOGRAMS = {}
|
||||
STATE = {'worker_busy': 0, 'queue_depth': 0, 'queue_oldest_age': 0, 'queue_observation_ok': 1,
|
||||
'spans_finished': 0, 'spans_exported': 0, 'spans_export_failed': 0}
|
||||
LOCK = threading.Lock()
|
||||
PROPAGATOR = TraceContextTextMapPropagator()
|
||||
LOGGER = logging.getLogger('messenger.telemetry')
|
||||
TRACER = trace.get_tracer('messenger')
|
||||
POOL = None
|
||||
SERVICE = ''
|
||||
BUCKETS = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 60]
|
||||
|
||||
class FinishedSpans(SpanProcessor):
|
||||
def on_end(self, span):
|
||||
with LOCK:
|
||||
STATE['spans_finished'] += 1
|
||||
|
||||
class CountedExporter(SpanExporter):
|
||||
def __init__(self, delegate):
|
||||
self.delegate = delegate
|
||||
|
||||
def export(self, spans):
|
||||
result = self.delegate.export(spans)
|
||||
with LOCK:
|
||||
STATE['spans_exported' if result == SpanExportResult.SUCCESS else 'spans_export_failed'] += len(spans)
|
||||
return result
|
||||
|
||||
def shutdown(self):
|
||||
self.delegate.shutdown()
|
||||
|
||||
def force_flush(self, timeout_millis=30000):
|
||||
return self.delegate.force_flush(timeout_millis)
|
||||
|
||||
def configure(service):
|
||||
global ACTIVE, TRACER, SERVICE
|
||||
if PROVIDERS or os.environ.get('TELEMETRY_ENABLED', '1') == '0':
|
||||
return
|
||||
SERVICE = service
|
||||
ratio = float(os.environ.get('TRACE_SAMPLE_RATIO', '0.25'))
|
||||
if not 0 <= ratio <= 1:
|
||||
raise ValueError('TRACE_SAMPLE_RATIO must be 0..1')
|
||||
resource_attributes = Resource.create({
|
||||
'service.name': service, 'service.version': 'lab6.1',
|
||||
'service.instance.id': INSTANCE, 'deployment.environment.name': 'lab',
|
||||
})
|
||||
endpoint = os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
|
||||
provider = TracerProvider(resource=resource_attributes, sampler=ParentBased(TraceIdRatioBased(ratio)))
|
||||
provider.add_span_processor(FinishedSpans())
|
||||
provider.add_span_processor(BatchSpanProcessor(
|
||||
CountedExporter(OTLPSpanExporter(endpoint=endpoint+'/v1/traces', timeout=1)),
|
||||
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=500,
|
||||
export_timeout_millis=1500))
|
||||
trace.set_tracer_provider(provider)
|
||||
TRACER = trace.get_tracer('messenger')
|
||||
reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=endpoint+'/v1/metrics', timeout=1),
|
||||
export_interval_millis=2000, export_timeout_millis=1500)
|
||||
meter_provider = MeterProvider(resource=resource_attributes, metric_readers=[reader],
|
||||
views=[View(instrument_name='*duration', aggregation=ExplicitBucketHistogramAggregation(BUCKETS))])
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
meter = metrics.get_meter('messenger')
|
||||
for name in ['http_server_requests', 'jobs_processed', 'job_retries']:
|
||||
COUNTERS[name] = meter.create_counter(name)
|
||||
for name in ['http_server_duration', 'job_processing_duration', 'job_ready_duration']:
|
||||
HISTOGRAMS[name] = meter.create_histogram(name, unit='s')
|
||||
def gauge(name, getter, unit=''):
|
||||
def observe(options):
|
||||
try:
|
||||
yield metrics.Observation(getter(), {})
|
||||
except (OSError, ValueError):
|
||||
return
|
||||
meter.create_observable_gauge(name, callbacks=[observe], unit=unit)
|
||||
for name in ['queue_depth', 'queue_oldest_age', 'queue_observation_ok', 'worker_busy',
|
||||
'spans_finished', 'spans_exported', 'spans_export_failed']:
|
||||
gauge(name, lambda name=name: STATE[name], 's' if name=='queue_oldest_age' else '')
|
||||
gauge('worker_capacity', lambda: int(os.environ.get('WORKER_CONCURRENCY', '1')) if service=='messenger-worker' else 0)
|
||||
gauge('db_pool_used', lambda: max(0, POOL.get_stats().get('pool_size',0)-POOL.get_stats().get('pool_available',0)) if POOL else 0)
|
||||
gauge('db_pool_waiting', lambda: POOL.get_stats().get('requests_waiting',0) if POOL else 0)
|
||||
gauge('db_pool_capacity', lambda: int(os.environ.get('DB_POOL_MAX','8')) if POOL else 0)
|
||||
gauge('process_resident_memory', lambda: int(Path('/proc/self/statm').read_text().split()[1])*os.sysconf('SC_PAGE_SIZE'), 'By')
|
||||
gauge('container_memory', lambda: int(Path('/sys/fs/cgroup/memory.current').read_text()), 'By')
|
||||
def cpu(options):
|
||||
use = resource.getrusage(resource.RUSAGE_SELF)
|
||||
yield metrics.Observation(use.ru_utime+use.ru_stime, {})
|
||||
meter.create_observable_counter('process_cpu', callbacks=[cpu], unit='s')
|
||||
log_provider = LoggerProvider(resource=resource_attributes)
|
||||
log_provider.add_log_record_processor(BatchLogRecordProcessor(
|
||||
OTLPLogExporter(endpoint=endpoint+'/v1/logs', timeout=1),
|
||||
max_queue_size=256, max_export_batch_size=64, schedule_delay_millis=500,
|
||||
export_timeout_millis=1500))
|
||||
LOGGER.setLevel(logging.INFO)
|
||||
LOGGER.propagate = False
|
||||
LOGGER.addHandler(LoggingHandler(level=logging.INFO, logger_provider=log_provider))
|
||||
PROVIDERS.extend([provider, meter_provider, log_provider])
|
||||
ACTIVE = True
|
||||
|
||||
def pool(value):
|
||||
global POOL
|
||||
POOL = value
|
||||
|
||||
def route(path):
|
||||
if path in ['/health/live', '/health/ready', '/api/v1/users', '/api/v1/users/me',
|
||||
'/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats']:
|
||||
return path
|
||||
patterns = [
|
||||
(r'/api/v1/chats/[^/]+/messages', '/api/v1/chats/{chat_id}/messages'),
|
||||
(r'/api/v1/chats/[^/]+/attachments', '/api/v1/chats/{chat_id}/attachments'),
|
||||
(r'/api/v1/chats/[^/]+', '/api/v1/chats/{chat_id}'),
|
||||
(r'/api/v1/attachments/[^/]+/content', '/api/v1/attachments/{attachment_id}/content'),
|
||||
(r'/api/v1/attachments/[^/]+', '/api/v1/attachments/{attachment_id}'),
|
||||
]
|
||||
return next((template for pattern,template in patterns if re.fullmatch(pattern,path)), 'unmatched')
|
||||
|
||||
@contextlib.contextmanager
|
||||
def span(name, attributes=None, kind=SpanKind.INTERNAL, carrier=None):
|
||||
if not ACTIVE or SUPPRESSED.get() or (name == 'db.transaction' and not trace.get_current_span().get_span_context().is_valid):
|
||||
yield trace.get_current_span()
|
||||
return
|
||||
context = PROPAGATOR.extract(carrier) if carrier is not None else None
|
||||
with TRACER.start_as_current_span(name, attributes=attributes or {}, kind=kind,
|
||||
context=context, record_exception=False, set_status_on_exception=False) as current:
|
||||
try:
|
||||
yield current
|
||||
except Exception as error:
|
||||
current.set_attribute('error.type', type(error).__name__)
|
||||
current.set_status(Status(StatusCode.ERROR))
|
||||
log('operation_failed', error_type=type(error).__name__)
|
||||
raise
|
||||
|
||||
@contextlib.contextmanager
|
||||
def suppressed():
|
||||
token = SUPPRESSED.set(True)
|
||||
try:
|
||||
yield None
|
||||
finally:
|
||||
SUPPRESSED.reset(token)
|
||||
|
||||
def carrier():
|
||||
result = {}
|
||||
if ACTIVE:
|
||||
PROPAGATOR.inject(result)
|
||||
return result
|
||||
|
||||
def log(event, **attributes):
|
||||
if not ACTIVE:
|
||||
return
|
||||
# Callers pass only fixed event names, bounded outcomes and technical correlation IDs.
|
||||
context = trace.get_current_span().get_span_context()
|
||||
body = {'timestamp': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'severity': 'INFO', 'service.name': SERVICE, 'event': event,
|
||||
'sampled': bool(context.trace_flags.sampled), 'trace_id': format(context.trace_id,'032x'), 'span_id': format(context.span_id,'016x'),
|
||||
**attributes}
|
||||
LOGGER.info(json.dumps(body, ensure_ascii=True, separators=(',',':')))
|
||||
|
||||
def count(name, attributes=None):
|
||||
if ACTIVE:
|
||||
COUNTERS[name].add(1, attributes or {})
|
||||
|
||||
def duration(name, seconds, attributes=None):
|
||||
if ACTIVE:
|
||||
HISTOGRAMS[name].record(max(0, seconds), attributes or {})
|
||||
|
||||
def flush():
|
||||
for provider in PROVIDERS:
|
||||
provider.force_flush(timeout_millis=5000)
|
||||
|
||||
def shutdown():
|
||||
for provider in PROVIDERS:
|
||||
provider.shutdown()
|
||||
|
||||
class S3Client:
|
||||
"""Span each SDK request without serializing args, keys, URLs or SDK exceptions."""
|
||||
def __init__(self, client):
|
||||
self.client = client
|
||||
|
||||
def __getattr__(self, name):
|
||||
method = getattr(self.client,name)
|
||||
if name not in {'head_object','get_object','put_object','delete_object','head_bucket','create_bucket','list_objects_v2'}:
|
||||
return method
|
||||
def call(*args, **kwargs):
|
||||
with span('s3.'+name, {'rpc.system':'aws-api','rpc.service':'S3','rpc.method':name}, SpanKind.CLIENT):
|
||||
return method(*args, **kwargs)
|
||||
return call
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Stop worker, accept queued, restart API/broker, then complete the same attachment."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
|
||||
from client import Client,Failure
|
||||
|
||||
|
||||
def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip()
|
||||
|
||||
|
||||
def main():
|
||||
c=Client();alice,bob,eve,chat=c.bootstrap();record={}
|
||||
before=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True))
|
||||
try:
|
||||
docker('compose','stop','worker','dispatcher')
|
||||
queued,raw=c.attachment(alice,chat)
|
||||
assert queued['status']=='queued'
|
||||
message=c.message(alice,chat,'accepted with worker stopped',attachments=[queued['id']])['json']
|
||||
c.call('GET','/api/v1/attachments/'+queued['id']+'/content',409,actor=bob)
|
||||
killed=docker('compose','ps','-q','api').splitlines()
|
||||
for container in killed:docker('kill','--signal=KILL',container)
|
||||
record['killed_api_containers']=killed
|
||||
docker('compose','up','-d','--wait','--no-deps','api')
|
||||
time.sleep(2)
|
||||
assert c.call('GET','/api/v1/attachments/'+queued['id'],actor=bob,schema='Attachment')['json']['status']=='queued'
|
||||
docker('compose','up','-d','--no-deps','dispatcher')
|
||||
deadline=time.monotonic()+10
|
||||
while int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))<1:
|
||||
assert time.monotonic()<deadline;time.sleep(.1)
|
||||
docker('compose','stop','dispatcher')
|
||||
length_before=int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))
|
||||
docker('compose','restart','broker')
|
||||
docker('compose','up','-d','--wait','--no-deps','broker')
|
||||
length_after=int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))
|
||||
assert length_after>=length_before>=1
|
||||
record['stream_entries_before_after_broker_restart']=[length_before,length_after]
|
||||
docker('compose','up','-d','--no-deps','worker','dispatcher')
|
||||
ready=c.wait_ready(bob,queued)
|
||||
assert c.call('GET','/api/v1/attachments/'+queued['id']+'/content',actor=bob)['bytes']==raw
|
||||
record.update(attachment_id=queued['id'],original_sha256=hashlib.sha256(raw).hexdigest(),status=ready['status'],variants=[v['name'] for v in ready['variants']],old_bearer_survived=True,message_attachment_retained=message['attachment_ids']==[queued['id']])
|
||||
finally:docker('compose','up','-d','--no-deps','worker','dispatcher')
|
||||
after=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True))
|
||||
record['before']=before;record['after']=after
|
||||
print(json.dumps(record,indent=2))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Small deterministic PNG helpers; generation happens outside measurement."""
|
||||
import random
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
|
||||
def chunk(name, data):
|
||||
return struct.pack('!I', len(data)) + name + data + struct.pack('!I', zlib.crc32(name + data) & 0xffffffff)
|
||||
|
||||
|
||||
def png(width, height, noise=False, rgb=True):
|
||||
channels = 3 if rgb else 1
|
||||
rand = random.Random(2301)
|
||||
encoded = zlib.compressobj(1)
|
||||
parts = []
|
||||
for _ in range(height):
|
||||
line = bytes(rand.getrandbits(8) for _ in range(width * channels)) if noise else b'\x7f' * (width * channels)
|
||||
parts.append(encoded.compress(b'\0' + line))
|
||||
parts.append(encoded.flush())
|
||||
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2 if rgb else 0, 0, 0, 0)) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'')
|
||||
|
||||
|
||||
def bomb():
|
||||
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', 10000, 4001, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'x')) + chunk(b'IEND', b'')
|
||||
@@ -0,0 +1,257 @@
|
||||
"""Real OTLP evidence, controlled incidents and bounded telemetry-backend outage."""
|
||||
import argparse
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
BASE=os.environ.get('BASE_URL','http://127.0.0.1:18093')
|
||||
PROM=os.environ.get('PROMETHEUS_URL','http://127.0.0.1:19096')
|
||||
LOKI=os.environ.get('LOKI_URL','http://127.0.0.1:13106')
|
||||
TEMPO=os.environ.get('TEMPO_URL','http://127.0.0.1:13206')
|
||||
COLLECTOR=os.environ.get('COLLECTOR_URL','http://127.0.0.1:18896')
|
||||
OPENER=urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def get(url):
|
||||
req=urllib.request.Request(url,headers={'Accept':'application/json'})
|
||||
with OPENER.open(req,timeout=10) as response:
|
||||
return json.load(response)
|
||||
|
||||
def prom(query):
|
||||
return get(PROM+'/api/v1/query?'+urlencode({'query':query}))['data']['result']
|
||||
|
||||
def scalar(query):
|
||||
rows=prom(query)
|
||||
return float(rows[0]['value'][1]) if rows else 0
|
||||
|
||||
def logs(query, start=None, limit=2000):
|
||||
params={'query':query,'start':str(int((start or time.time()-600)*1e9)),
|
||||
'end':str(time.time_ns()),'limit':str(limit),'direction':'forward'}
|
||||
result=get(LOKI+'/loki/api/v1/query_range?'+urlencode(params))['data']['result']
|
||||
return [json.loads(value) for stream in result for stamp,value in stream['values']]
|
||||
|
||||
def trace(identifier):
|
||||
return get(TEMPO+'/api/traces/'+identifier)
|
||||
|
||||
def spans(data):
|
||||
result=[]
|
||||
for batch in data.get('batches',data.get('resourceSpans',[])):
|
||||
service=next(a['value']['stringValue'] for a in batch['resource']['attributes'] if a['key']=='service.name')
|
||||
for scope in batch.get('scopeSpans',[]):
|
||||
for span in scope['spans']:
|
||||
result.append(dict(span,service=service))
|
||||
return result
|
||||
|
||||
def wait(function,timeout=60):
|
||||
end=time.monotonic()+timeout
|
||||
while time.monotonic()<end:
|
||||
try:
|
||||
result=function()
|
||||
if result:return result
|
||||
except (OSError,ValueError,Failure):
|
||||
pass
|
||||
time.sleep(.5)
|
||||
raise AssertionError('Timed out: '+function.__name__)
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.check_output(['docker','compose',*args],text=True).strip()
|
||||
|
||||
def save(name,data):
|
||||
(ROOT/'evidence'/name).write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n')
|
||||
|
||||
def upload(c,actor,chat,tid):
|
||||
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=private-canary.png',202,
|
||||
actor=actor,raw=raw,headers={'Content-Type':'image/png',
|
||||
'traceparent':'00-'+tid+'-0123456789abcdef-01'},schema='Attachment')
|
||||
|
||||
def proof():
|
||||
wait(lambda:get(LOKI+'/loki/api/v1/labels'))
|
||||
wait(lambda:get(TEMPO+'/api/search'))
|
||||
proof_start=time.time()
|
||||
c=Client(BASE);a,b,e,chat=c.bootstrap()
|
||||
tid=uuid.uuid4().hex
|
||||
marker='SECRET_MESSAGE_CANARY_'+uuid.uuid4().hex
|
||||
c.message(a,chat,marker)
|
||||
response=upload(c,a,chat,tid)
|
||||
ready=c.wait_ready(b,response['json'])
|
||||
def completed_trace():
|
||||
data=trace(tid)
|
||||
return data if any(s['name']=='attachment process' for s in spans(data)) else None
|
||||
data=wait(completed_trace)
|
||||
rows=spans(data)
|
||||
upload_span=next(s for s in rows if s['kind']=='SPAN_KIND_SERVER')
|
||||
publish=next(s for s in rows if s['kind']=='SPAN_KIND_PRODUCER')
|
||||
consumer=next(s for s in rows if s['kind']=='SPAN_KIND_CONSUMER')
|
||||
assert consumer['parentSpanId']==publish['spanId']
|
||||
index={s['spanId']:s for s in rows}
|
||||
parent=publish
|
||||
while parent['spanId']!=upload_span['spanId']:
|
||||
parent=index[parent['parentSpanId']]
|
||||
assert any(s['service']=='messenger-worker' and s['name'].startswith('s3.') for s in rows)
|
||||
assert any(s['service']=='messenger-worker' and s['name']=='db.transaction' for s in rows)
|
||||
query='{service_name=~"messenger-.*"} | json | trace_id="'+tid+'"'
|
||||
records=wait(lambda: logs(query) if len(logs(query))>=4 else None)
|
||||
assert {r['event'] for r in records}>={'http_request','attachment_accepted','job_published','job_processed'}
|
||||
assert any(r.get('request_id')==response['headers'].get('X-Request-Id') for r in records)
|
||||
assert all(len(r['trace_id'])==32 and len(r['span_id'])==16 for r in records)
|
||||
all_logs=logs('{service_name=~"messenger-.*"}',start=proof_start,limit=5000)
|
||||
payload=json.dumps(all_logs)+json.dumps(data)
|
||||
for value in [marker,a['token'],a['registration']['password'],a['registration']['username'],'private-canary.png']:
|
||||
assert value not in payload, 'business/credential canary leaked'
|
||||
metric_rows=prom('{service_name=~"messenger-.*"}')
|
||||
forbidden={'user_id','chat_id','message_id','attachment_id','request_id','filename','url','job_id','trace_id','span_id'}
|
||||
assert all(not forbidden.intersection(row['metric']) for row in metric_rows)
|
||||
names={r['metric']['__name__'] for r in metric_rows}
|
||||
assert {'http_server_duration_seconds_bucket','jobs_processed_total','queue_depth','queue_oldest_age_seconds','db_pool_used','worker_capacity'}<=names
|
||||
save('trace-proof.json',{'trace_id':tid,'request_id':response['headers'].get('X-Request-Id'),
|
||||
'status':ready['status'],'consumer_parent_matches_producer':True,'producer_descends_from_upload':True,
|
||||
'api_worker_logs_correlated':True,'secret_canaries_absent':True,'metric_labels_allowlisted':True,
|
||||
'trace':data,'logs':records})
|
||||
# Retry is a real S3 outage after durable acceptance, not fabricated spans.
|
||||
compose('stop','worker')
|
||||
retry_tid=uuid.uuid4().hex
|
||||
queued=upload(c,a,chat,retry_tid)['json']
|
||||
paused=False
|
||||
try:
|
||||
compose('pause','s3');paused=True
|
||||
compose('up','-d','--no-deps','worker')
|
||||
def attempted():
|
||||
raw=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',
|
||||
"SELECT attempt FROM attachment_jobs WHERE 1=1 AND attachment_id = '"+queued['id']+"';")
|
||||
return int(raw)>=1
|
||||
wait(attempted)
|
||||
time.sleep(5)
|
||||
finally:
|
||||
if paused:compose('unpause','s3')
|
||||
compose('up','-d','--no-deps','worker')
|
||||
wait(lambda:c.call('GET','/health/ready',schema='Health'))
|
||||
c.wait_ready(a,queued)
|
||||
def retried():
|
||||
data=trace(retry_tid)
|
||||
return data if len([s for s in spans(data) if s['kind']=='SPAN_KIND_CONSUMER'])>=2 else None
|
||||
retry_data=wait(retried)
|
||||
retry_records=wait(lambda:logs('{service_name="messenger-worker"} | json | trace_id="'+retry_tid+'" | event="job_retry"'))
|
||||
save('retry-trace.json',{'trace_id':retry_tid,'retry_observed':True,'trace':retry_data,'logs':retry_records})
|
||||
print('PASS: actual API/producer/consumer/S3/DB chain, correlated safe logs, real retry and low cardinality.')
|
||||
|
||||
def alerts():
|
||||
return get(PROM+'/api/v1/alerts')['data']['alerts']
|
||||
|
||||
def firing(name):
|
||||
return [a for a in alerts() if a['labels']['alertname']==name and a['state']=='firing']
|
||||
|
||||
def incidents():
|
||||
c=Client(BASE);a,b,e,chat=c.bootstrap();result={}
|
||||
start=time.time()
|
||||
compose('stop','worker')
|
||||
try:
|
||||
queued=[c.attachment(a,chat)[0] for _ in range(3)]
|
||||
observed=wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')>=3)
|
||||
fired=wait(lambda:firing('QueueLag'))
|
||||
detected=time.time()
|
||||
metrics={q:prom(q) for q in ['queue_depth','queue_oldest_age_seconds','worker_busy','worker_capacity']}
|
||||
result['worker_stop']={'injected_at':start,'detected_at':detected,
|
||||
'detection_seconds':detected-start,'firing':fired,'metrics':metrics,
|
||||
'diagnosis':'Backlog and oldest age grow; worker completion stops while API acceptance still succeeds. Gauges expire after 30s.'}
|
||||
finally:
|
||||
compose('up','-d','--no-deps','worker')
|
||||
for att in queued:c.wait_ready(a,att)
|
||||
wait(lambda:scalar('max(queue_depth{service_name="messenger-dispatcher"})')==0)
|
||||
wait(lambda:not firing('QueueLag'))
|
||||
result['worker_stop']['resolved_at']=time.time()
|
||||
# Keep readiness SELECT 1 available while real session SQL is blocked.
|
||||
lock=subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1',
|
||||
'-c',"BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(35); COMMIT;"],
|
||||
stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
time.sleep(1)
|
||||
db_start=time.time();stop=False;errors=[];error_trace=uuid.uuid4().hex
|
||||
def traffic():
|
||||
while not stop:
|
||||
r=c.call('GET','/api/v1/users/me',[200,503],actor=a,
|
||||
headers={'traceparent':'00-'+error_trace+'-0123456789abcdef-01'})
|
||||
if r['status']==503:errors.append(r['elapsed'])
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
futures=[pool.submit(traffic) for _ in range(4)]
|
||||
try:
|
||||
fired=wait(lambda:firing('APIHighErrorRate'),timeout=50)
|
||||
detected=time.time()
|
||||
result['db_lock']={'injected_at':db_start,'detected_at':detected,'detection_seconds':detected-db_start,
|
||||
'firing':fired,'error_latencies_seconds':errors.copy(),'trace_id':error_trace}
|
||||
lock.wait(timeout=45)
|
||||
finally:
|
||||
stop=True
|
||||
for future in futures:future.result()
|
||||
finally:
|
||||
stop=True
|
||||
if lock.poll() is None:
|
||||
lock.terminate();lock.wait(timeout=5)
|
||||
wait(lambda:c.call('GET','/api/v1/users/me',actor=a,schema='User'))
|
||||
wait(lambda:not firing('APIHighErrorRate'),timeout=70)
|
||||
data=wait(lambda:trace(error_trace))
|
||||
assert any(s['name']=='db.transaction' and s.get('status',{}).get('code')=='STATUS_CODE_ERROR' for s in spans(data))
|
||||
result['db_lock'].update(resolved_at=time.time(),trace=data,
|
||||
diagnosis='Server 5xx spans contain slow/error PostgreSQL transaction; health remains available; sessions lock is the bottleneck.')
|
||||
save('incidents.json',result)
|
||||
print('PASS: worker-stop and real DB-lock incidents, local firing/resolved and causal DB traces.')
|
||||
|
||||
def collector_metrics():
|
||||
with OPENER.open(COLLECTOR+'/metrics',timeout=5) as r:raw=r.read().decode()
|
||||
result={}
|
||||
for line in raw.splitlines():
|
||||
if line.startswith('otelcol_') and not line.startswith('#'):
|
||||
name=line.split('{')[0].split(' ')[0]
|
||||
try:result[name]=result.get(name,0)+float(line.rsplit(' ',1)[-1])
|
||||
except ValueError:pass
|
||||
return result
|
||||
|
||||
def backend_outage():
|
||||
c=Client(BASE);a,b,e,chat=c.bootstrap();before=collector_metrics()
|
||||
cid=compose('ps','-q','otel-collector')
|
||||
samples=[];messages=[];attachments=[];start=time.time()
|
||||
compose('stop','tempo')
|
||||
try:
|
||||
for i in range(45):
|
||||
for _ in range(3):
|
||||
messages.append(c.message(a,chat,'synthetic-backend-outage')['json']['id'])
|
||||
if i%10==0:
|
||||
att,_=c.attachment(a,chat);attachments.append(c.wait_ready(a,att)['id'])
|
||||
stat=json.loads(subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',cid],text=True))
|
||||
samples.append({'elapsed':time.time()-start,'memory':stat['MemUsage'],'cpu':stat['CPUPerc']})
|
||||
time.sleep(1)
|
||||
during=collector_metrics()
|
||||
assert during.get('otelcol_exporter_send_failed_spans_total',0)>before.get('otelcol_exporter_send_failed_spans_total',0)
|
||||
assert during.get('otelcol_exporter_queue_size',0)<=128*2
|
||||
history=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100',actor=b,schema='MessagePage')['json']
|
||||
got={m['id'] for m in history['items']}
|
||||
if history['next_cursor']:
|
||||
history2=c.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+history['next_cursor'],actor=b,schema='MessagePage')['json']
|
||||
got.update(m['id'] for m in history2['items'])
|
||||
assert set(messages)<=got
|
||||
finally:compose('up','-d','--no-deps','tempo')
|
||||
wait(lambda:get(TEMPO+'/api/search'))
|
||||
tid=uuid.uuid4().hex
|
||||
upload(c,a,chat,tid)
|
||||
wait(lambda:trace(tid))
|
||||
save('backend-outage.json',{'duration_seconds':time.time()-start,'confirmed_messages':len(messages),
|
||||
'confirmed_attachments_ready':len(attachments),'all_confirmed_messages_retained':True,
|
||||
'collector_samples':samples,'before':before,'during':during,'recovered_trace_id':tid})
|
||||
print('PASS: Tempo unavailable, finite Collector buffers/drops, API writes/jobs retained, fresh trace after recovery.')
|
||||
|
||||
if __name__=='__main__':
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('phase',choices=['proof','incidents','backend'])
|
||||
args=parser.parse_args()
|
||||
{'proof':proof,'incidents':incidents,'backend':backend_outage}[args.phase]()
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Fixed-arrival mixed workload: SDK off, full traces, 10% and 25% head sampling."""
|
||||
import concurrent.futures
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
||||
from observability import Client, BASE, compose, collector_metrics, scalar, prom, logs, wait, save
|
||||
|
||||
class DockerHTTP(http.client.HTTPConnection):
|
||||
def connect(self):
|
||||
self.sock=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM)
|
||||
self.sock.settimeout(8)
|
||||
self.sock.connect(DOCKER_SOCKET)
|
||||
|
||||
context=json.loads(subprocess.check_output(['docker','context','inspect'],text=True))[0]
|
||||
DOCKER_SOCKET=urlsplit(context['Endpoints']['docker']['Host']).path
|
||||
|
||||
def stats(container):
|
||||
connection=DockerHTTP('localhost')
|
||||
try:
|
||||
connection.request('GET','/containers/'+container+'/stats?stream=false&one-shot=true')
|
||||
response=connection.getresponse()
|
||||
assert response.status==200
|
||||
data=json.load(response)
|
||||
finally:connection.close()
|
||||
memory=data.get('memory_stats',{})
|
||||
details=memory.get('stats',{})
|
||||
usage=memory.get('usage',0)
|
||||
return {'cpu_seconds':data['cpu_stats']['cpu_usage']['total_usage']/1e9,
|
||||
'anonymous_bytes':details.get('anon',details.get('rss',0)),
|
||||
'working_set_bytes':max(0,usage-details.get('inactive_file',details.get('total_inactive_file',0))),
|
||||
'rx_bytes':sum(n.get('rx_bytes',0) for n in data.get('networks',{}).values())}
|
||||
|
||||
def span_counts():
|
||||
return {r['metric']['service_instance_id']:float(r['value'][1])
|
||||
for r in prom('spans_finished{service_name=~"messenger-.*"}')}
|
||||
|
||||
def delta(previous,current):
|
||||
return sum(max(0,value-previous.get(key,0)) for key,value in current.items())
|
||||
|
||||
def quantile(values,q):
|
||||
return sorted(values)[min(len(values)-1,int((len(values)-1)*q))]
|
||||
|
||||
def set_mode(enabled,ratio):
|
||||
env=os.environ.copy()
|
||||
env.update(TELEMETRY_ENABLED=str(enabled),TRACE_SAMPLE_RATIO=str(ratio))
|
||||
subprocess.run(['docker','compose','up','-d','--wait','--no-deps','api','worker','dispatcher'],env=env,
|
||||
check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
c=Client(BASE)
|
||||
wait(lambda:c.call('GET','/health/ready',schema='Health'))
|
||||
return c
|
||||
|
||||
def probes(c,actor,mode):
|
||||
results=[]
|
||||
for duration in [3.5,3.5,3.5,1.2]:
|
||||
query="BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep("+str(duration)+"); COMMIT;"
|
||||
lock=subprocess.Popen(['docker','compose','exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',query],
|
||||
stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
def locked():
|
||||
value=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',
|
||||
"SELECT count(*) FROM pg_locks WHERE 1=1 AND relation = 'sessions'::regclass AND mode = 'AccessExclusiveLock' AND granted;")
|
||||
return int(value)>0
|
||||
wait(locked,timeout=5)
|
||||
result=c.call('GET','/api/v1/users/me',[200,503],actor=actor)
|
||||
assert result['status']==(503 if duration>3 else 200)
|
||||
assert result['elapsed']>.1
|
||||
results.append({'request_id':result['headers'].get('X-Request-Id'),
|
||||
'status':result['status'],'latency_seconds':result['elapsed']})
|
||||
finally:lock.wait(timeout=5)
|
||||
if mode!='off':
|
||||
time.sleep(3)
|
||||
for row in results:
|
||||
records=logs('{service_name="messenger-api"} | json | request_id="'+row['request_id']+'"',limit=10)
|
||||
assert records
|
||||
row['trace_sampled']=records[0]['sampled']
|
||||
row['trace_id']=records[0]['trace_id']
|
||||
else:
|
||||
for row in results:row['trace_sampled']=False
|
||||
return results
|
||||
|
||||
def run(mode,enabled,ratio,repeat,seconds,rps):
|
||||
c=set_mode(enabled,ratio);a,b,e,chat=c.bootstrap()
|
||||
for _ in range(10):c.call('GET','/api/v1/users/me',actor=a)
|
||||
time.sleep(5)
|
||||
names=compose('ps','-q','api','worker','dispatcher','otel-collector','postgres','loki','tempo','prometheus','grafana','s3','s3-store','broker','lb','redis').splitlines()
|
||||
labels={}
|
||||
for cid in names:
|
||||
label=json.loads(subprocess.check_output(['docker','inspect',cid,'--format','{{json .Config.Labels}}'],text=True))
|
||||
labels[cid]=label['com.docker.compose.service']
|
||||
before={cid:stats(cid) for cid in names}
|
||||
ingest_before=collector_metrics();spans_before=span_counts()
|
||||
memory={cid:[] for cid in names};sampling_stop=threading.Event()
|
||||
def sample():
|
||||
while not sampling_stop.is_set():
|
||||
for cid in names:
|
||||
try:memory[cid].append(stats(cid))
|
||||
except OSError:pass
|
||||
sampling_stop.wait(1)
|
||||
sampler=threading.Thread(target=sample,daemon=True);sampler.start()
|
||||
latencies=[];failures=[];attachments=[];start=time.monotonic()
|
||||
def request(i):
|
||||
try:
|
||||
if i%20==0:
|
||||
result=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=cost.png',202,
|
||||
actor=a,raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes(),
|
||||
headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
return result['elapsed'],result['json']
|
||||
if i%5==0:
|
||||
result=c.message(a,chat,'fixed-synthetic-cost-payload')
|
||||
else:result=c.call('GET','/api/v1/users/me',actor=a,schema='User')
|
||||
return result['elapsed'],None
|
||||
except Exception as error:return None,type(error).__name__
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool:
|
||||
futures=[]
|
||||
for i in range(seconds*rps):
|
||||
delay=start+i/rps-time.monotonic()
|
||||
if delay>0:time.sleep(delay)
|
||||
futures.append(pool.submit(request,i))
|
||||
for future in futures:
|
||||
elapsed,att=future.result()
|
||||
if elapsed is None:failures.append(att)
|
||||
else:
|
||||
latencies.append(elapsed)
|
||||
if att:attachments.append(att)
|
||||
load_wall=time.monotonic()-start
|
||||
for att in attachments:c.wait_ready(a,att)
|
||||
# Drain exporter batches and metric-reader intervals, then compare counters by instance.
|
||||
time.sleep(5)
|
||||
sampling_stop.set();sampler.join(timeout=15)
|
||||
after={cid:stats(cid) for cid in names}
|
||||
ingest_after=collector_metrics();spans_after=span_counts()
|
||||
assert not failures, failures
|
||||
resource={}
|
||||
for cid in names:
|
||||
service=labels[cid]
|
||||
target=resource.setdefault(service,{'cpu_seconds':0,'peak_sampled_anonymous_bytes':0,'peak_sampled_working_set_bytes':0,'rx_bytes':0})
|
||||
target['cpu_seconds']+=max(0,after[cid]['cpu_seconds']-before[cid]['cpu_seconds'])
|
||||
target['rx_bytes']+=max(0,after[cid]['rx_bytes']-before[cid]['rx_bytes'])
|
||||
target['peak_sampled_anonymous_bytes']+=max((s['anonymous_bytes'] for s in memory[cid]),default=0)
|
||||
target['peak_sampled_working_set_bytes']+=max((s['working_set_bytes'] for s in memory[cid]),default=0)
|
||||
counts={k:max(0,ingest_after.get(k,0)-ingest_before.get(k,0)) for k in [
|
||||
'otelcol_receiver_accepted_spans_total','otelcol_receiver_accepted_log_records_total',
|
||||
'otelcol_receiver_accepted_metric_points_total','otelcol_exporter_sent_spans_total',
|
||||
'otelcol_exporter_send_failed_spans_total']}
|
||||
finished=delta(spans_before,spans_after)
|
||||
accepted=counts['otelcol_receiver_accepted_spans_total']
|
||||
result={'mode':mode,'enabled':enabled,'sample_ratio':ratio,'repeat':repeat,
|
||||
'arrival_rps':rps,'scheduled_requests':seconds*rps,'confirmed_requests':len(latencies),
|
||||
'load_wall_seconds':load_wall,'successful_rps':len(latencies)/load_wall,
|
||||
'p50_seconds':quantile(latencies,.5),'p95_seconds':quantile(latencies,.95),'p99_seconds':quantile(latencies,.99),
|
||||
'errors':failures,'image_jobs_ready':len(attachments),'resource':resource,'collector_ingest':counts,
|
||||
'sampled_spans_finished':finished,'sampled_spans_not_accepted_after_drain':max(0,finished-accepted)}
|
||||
result['rare_probes']=probes(c,a,mode)
|
||||
print(mode,repeat,'p95',round(result['p95_seconds'],5),'ingested spans',accepted,flush=True)
|
||||
return result
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--seconds',type=int,default=20)
|
||||
parser.add_argument('--rps',type=int,default=30)
|
||||
parser.add_argument('--repeats',type=int,default=3)
|
||||
args=parser.parse_args()
|
||||
result={'method':'Fixed arrival mixed requests, 5% image uploads, 15% messages, 80% profile reads; same fixture/resources. No transparent retries.',
|
||||
'notes':['CPU cgroup total across load plus exporter drain; includes renderer.',
|
||||
'Anonymous/working-set memory sampled at 1Hz through read-only Docker stats API, not exact per-PID peak RSS.',
|
||||
'Collector rx bytes include OTLP HTTP, Prometheus scrapes and backend replies; accepted signal counts are exact Collector counter deltas.',
|
||||
'Fresh API/worker/dispatcher containers per setting; data/backends retained. Per-instance counter deltas ignore expired old series.',
|
||||
'Rare SQL-lock probes run after resource/ingest measurement; three SQL-lock probes and one slow success per run; future runs synchronize on pg_locks.'],
|
||||
'runs':[]}
|
||||
modes=[('off',0,0),('full',1,1),('sample10',1,.1),('sample25',1,.25)]
|
||||
try:
|
||||
for repeat in range(args.repeats):
|
||||
# Rotate order to reduce a fixed warm-cache/time-order advantage.
|
||||
order=modes[repeat%len(modes):]+modes[:repeat%len(modes)]
|
||||
for mode,enabled,ratio in order:
|
||||
result['runs'].append(run(mode,enabled,ratio,repeat,args.seconds,args.rps))
|
||||
save('observability-cost.json',result)
|
||||
finally:set_mode(1,.25)
|
||||
print('PASS: telemetry cost experiment completed; balanced 25% tracing restored.')
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Restart all stores and verify historical telemetry alongside public business persistence."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
from observability import BASE,PROM,trace,spans,logs,get,wait,save
|
||||
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
identifier=json.loads((root/'evidence/trace-proof.json').read_text())['trace_id']
|
||||
before_trace=trace(identifier)
|
||||
before_ids={s['spanId'] for s in spans(before_trace)}
|
||||
query='{service_name=~"messenger-.*"} | json | trace_id="'+identifier+'"'
|
||||
before_logs=logs(query,start=time.time()-3600)
|
||||
assert before_logs
|
||||
at=time.time()-5
|
||||
url=PROM+'/api/v1/query?'+urlencode({'query':'sum(http_server_requests_total)','time':str(at)})
|
||||
before_metric=get(url)['data']['result']
|
||||
assert before_metric
|
||||
command=['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh','BASE_URL='+BASE]
|
||||
result=subprocess.run(command,capture_output=True,text=True)
|
||||
(root/'evidence/persistence.txt').write_text(result.stdout+result.stderr)
|
||||
assert result.returncode==0,result.stdout[-300:]
|
||||
after_trace=wait(lambda:trace(identifier),timeout=90)
|
||||
after_ids={s['spanId'] for s in spans(after_trace)}
|
||||
assert before_ids<=after_ids
|
||||
after_logs=wait(lambda:logs(query,start=time.time()-3600),timeout=90)
|
||||
assert {json.dumps(r,sort_keys=True) for r in before_logs}<={json.dumps(r,sort_keys=True) for r in after_logs}
|
||||
after_metric=wait(lambda:get(url)['data']['result'])
|
||||
assert before_metric==after_metric
|
||||
save('telemetry-persistence.json',{'trace_id':identifier,'historical_span_count':len(before_ids),
|
||||
'historical_logs':len(before_logs),'fixed_metric_query_time':at,
|
||||
'metric_before':before_metric,'metric_after':after_metric,
|
||||
'business_data_and_existing_session_survived':True,'all_telemetry_volumes_restarted':True,
|
||||
'trace_log_metric_history_survived':True})
|
||||
print('PASS: full business restart, original Bearer, historical Tempo trace, Loki logs and fixed-time Prometheus sample.')
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Real DB/S3/Redis fault injection. Stop production worker/dispatcher first."""
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
sys.path[:0]=['/app/src',str(Path.cwd()/'tests')]
|
||||
from PIL import Image
|
||||
from client import Client
|
||||
from pipeline import STREAM,GROUP,broker,connect,event,dispatch_once,process,ack
|
||||
from objects import Objects
|
||||
from postgres import setting
|
||||
from admin import replay,version,cleanup
|
||||
|
||||
|
||||
def main():
|
||||
c=Client();alice,bob,eve,chat=c.bootstrap()
|
||||
bus=broker(); objects=Objects(setting)
|
||||
report={'invariants':{},'faults':[]}
|
||||
def fetch(aid):
|
||||
with connect() as conn:
|
||||
a=conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s',(aid,)).fetchone()
|
||||
j=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(a['current_job_id'],)).fetchone()
|
||||
return a,j
|
||||
def upload(raw=None,mime='image/png'):
|
||||
if raw is None: a,_=c.attachment(alice,chat)
|
||||
else:a=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=research.png',202,actor=alice,raw=raw,headers={'Content-Type':mime},schema='Attachment')['json']
|
||||
return a,fetch(a['id'])[1]
|
||||
def run_once(fault=None):
|
||||
args=[sys.executable,'/app/src/pipeline.py','work','--once']
|
||||
if fault:args+=['--fault-point',fault]
|
||||
return subprocess.run(args,capture_output=True,timeout=30).returncode
|
||||
def drain():
|
||||
for _ in range(200):
|
||||
batch=bus.xreadgroup(GROUP,'research-drain',{STREAM:'>'},count=1)
|
||||
if not batch:break
|
||||
for mid,data in batch[0][1]:process(json.loads(data['event']));ack(bus,mid)
|
||||
def expire(job):
|
||||
with connect() as conn:
|
||||
conn.execute("UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE 1=1 AND id = %s",(job['id'],))
|
||||
def recover(aid):
|
||||
until=time.monotonic()+20
|
||||
while time.monotonic()<until:
|
||||
dispatch_once(bus);drain()
|
||||
a,j=fetch(aid)
|
||||
if a['state']=='ready':return a,j
|
||||
time.sleep(.1)
|
||||
raise AssertionError('not recovered')
|
||||
for mid,data in bus.xautoclaim(STREAM,GROUP,'initial-recovery',min_idle_time=0,start_id='0-0',count=100)[1]:
|
||||
process(json.loads(data['event']));ack(bus,mid)
|
||||
# COMMIT -> no publisher yet: one queued row, no Redis message, then automatic outbox pass.
|
||||
a,j=upload()
|
||||
assert fetch(a['id'])[0]['state']=='queued'
|
||||
assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',409,actor=bob)
|
||||
c.message(bob,chat,'queued is linkable',attachments=[a['id']])
|
||||
dispatch_once(bus);drain();assert fetch(a['id'])[0]['state']=='ready'
|
||||
report['invariants']['commit_before_publish_recovered']=True
|
||||
# Broker accepted XADD but publisher died before marking outbox: true exit(77).
|
||||
a,j=upload()
|
||||
code=subprocess.run([sys.executable,'/app/src/pipeline.py','dispatch','--once','--fault-point','after_publish'],capture_output=True,timeout=10).returncode
|
||||
assert code==77
|
||||
dispatch_once(bus);drain()
|
||||
assert fetch(a['id'])[0]['state']=='ready'
|
||||
report['invariants']['publish_before_marker_duplicate_safe']=True
|
||||
for point in ['before_side_effect','after_original','after_objects','after_commit']:
|
||||
a,j=upload();dispatch_once(bus)
|
||||
code=run_once(point);assert code==77,(point,code)
|
||||
before,job=fetch(a['id'])
|
||||
if point not in ('after_commit','before_side_effect'):expire(job)
|
||||
result,job=recover(a['id'])
|
||||
assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',actor=bob)['bytes']==(Path.cwd()/'tests/fixtures/sample.png').read_bytes()
|
||||
reclaimed=bus.xautoclaim(STREAM,GROUP,'post-crash-redelivery',min_idle_time=0,start_id='0-0',count=100)[1]
|
||||
for mid,data in reclaimed:
|
||||
process(json.loads(data['event']));ack(bus,mid)
|
||||
assert bus.xpending(STREAM,GROUP)['pending']==0
|
||||
keys=objects.client.list_objects_v2(Bucket=objects.bucket,Prefix='results/'+a['id']+'/v1/').get('Contents',[])
|
||||
assert len(keys)==4,(point,len(keys))
|
||||
report['faults'].append({'point':point,'exit_code':code,'state_at_crash':before['state'],'recovered':result['state'],'attempts':job['attempt'],'result_objects':len(keys)})
|
||||
report['invariants']['crashes_before_after_side_effect_and_commit']=True
|
||||
bus.xadd(STREAM,{'event':json.dumps({'not':'a job'})})
|
||||
assert run_once()==0
|
||||
assert bus.xlen('rejected-jobs')>=1
|
||||
report['invariants']['invalid_envelope_observable_dlq']=True
|
||||
# Duplicate/reorder through broker on a completed job: no extra attempt/effect.
|
||||
a,j=upload();dispatch_once(bus);drain();before,done=fetch(a['id'])
|
||||
old=event(j)
|
||||
for attempt in [2,1,3,1,1]:bus.xadd(STREAM,{'event':json.dumps(dict(old,attempt=attempt))})
|
||||
drain();after,done2=fetch(a['id'])
|
||||
assert before['variants']==after['variants'] and done2['attempt']==done['attempt']
|
||||
report['invariants']['duplicate_and_reorder_no_second_effect']=True
|
||||
# Race two consumers with the SAME logical job, not just one serial duplicate.
|
||||
a,j=upload();e=event(j)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: results=list(pool.map(lambda _:process(e),range(2)))
|
||||
assert fetch(a['id'])[0]['state']=='ready' and sorted(results)==['noop','ready']
|
||||
report['invariants']['concurrent_duplicate_single_claim']=True
|
||||
# Pause actual old process after all immutable side effects, advance version and finish v2.
|
||||
a,j=upload();e=event(j)
|
||||
directory=Path('/tmp')/('fence-'+uuid.uuid4().hex);directory.mkdir();marker=directory/'paused';release=directory/'release'
|
||||
code="import sys,json,time;from pathlib import Path;sys.path.insert(0,'/app/src');from pipeline import process\ndef hook(point,job):\n if point=='after_objects':\n Path(sys.argv[2]).write_text('paused')\n while not Path(sys.argv[3]).exists():time.sleep(.05)\nprint(process(json.loads(sys.argv[1]),hook))"
|
||||
child=subprocess.Popen([sys.executable,'-c',code,json.dumps(e),str(marker),str(release)],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
|
||||
try:
|
||||
deadline=time.monotonic()+20
|
||||
while not marker.exists():
|
||||
assert child.poll() is None
|
||||
assert time.monotonic()<deadline;time.sleep(.05)
|
||||
new=version(uuid.UUID(a['id']),2);newrow,newjob=fetch(a['id'])
|
||||
assert process(event(newjob))=='ready'
|
||||
current,finished=fetch(a['id']);signature=current['variants']
|
||||
release.write_text('go');stdout,stderr=child.communicate(timeout=10)
|
||||
assert child.returncode==0 and stdout.strip()=='fenced',(stdout,stderr)
|
||||
bus.xadd(STREAM,{'event':json.dumps(e)});drain()
|
||||
final,finished=fetch(a['id'])
|
||||
assert final['pipeline_version']==2 and final['variants']==signature and final['state']=='ready'
|
||||
assert all('/v2/' in v['object_key'] for v in final['variants'])
|
||||
report['invariants']['stale_running_version_fenced']=True
|
||||
report['invariants']['old_delivery_after_new_ready_no_overwrite']=True
|
||||
finally:
|
||||
release.write_text('go')
|
||||
if child.poll() is None:child.kill();child.wait()
|
||||
# Transient failures consume a finite budget; failed registry and explicit replay.
|
||||
a,j=upload()
|
||||
with connect() as conn:conn.execute('UPDATE attachment_jobs SET max_attempts = 2 WHERE 1=1 AND id = %s',(j['id'],))
|
||||
# Exact staging copy saved externally; missing source is permanent and observable.
|
||||
raw=(Path.cwd()/'tests/fixtures/sample.png').read_bytes();objects.delete(j['source_key'])
|
||||
process(event(j));failed,fj=fetch(a['id']);assert failed['state']=='failed' and failed['error_code']=='source_missing'
|
||||
objects.put(j['source_key'],io.BytesIO(raw),len(raw),'image/png',hashlib.sha256(raw).hexdigest())
|
||||
replay(j['id']);fresh,fj=fetch(a['id']);assert process(event(fj))=='ready'
|
||||
try:replay(j['id']);raise AssertionError('replayed ready')
|
||||
except ValueError:pass
|
||||
report['invariants']['failed_registry_explicit_replay_and_ready_refusal']=True
|
||||
# Real S3 signature failures are transient, with two attempts and measured backoff.
|
||||
a,j=upload()
|
||||
with connect() as conn:conn.execute('UPDATE attachment_jobs SET max_attempts = 2 WHERE 1=1 AND id = %s',(j['id'],))
|
||||
wrong=Path('/tmp/wrong-secret');wrong.write_text('synthetic-invalid-key-not-a-real-secret')
|
||||
env=dict(os.environ,S3_SECRET_ACCESS_KEY_FILE=str(wrong))
|
||||
transitions=[];started=time.monotonic()
|
||||
for _ in range(2):
|
||||
row,j=fetch(a['id'])
|
||||
delay=max(0,(j['next_attempt_at']-__import__('datetime').datetime.now(__import__('datetime').timezone.utc)).total_seconds())
|
||||
time.sleep(delay+.01)
|
||||
code="import sys,json;sys.path.insert(0,'/app/src');from pipeline import process;print(process(json.loads(sys.argv[1])))"
|
||||
run=subprocess.run([sys.executable,'-c',code,json.dumps(event(j))],env=env,capture_output=True,timeout=10)
|
||||
assert run.returncode==0
|
||||
row,j=fetch(a['id']);transitions.append({'state':j['state'],'attempt':j['attempt'],'error_code':j['error_code']})
|
||||
assert row['state']=='failed' and row['error_code']=='retry_exhausted' and j['attempt']==2
|
||||
replay(j['id']);row,j=fetch(a['id']);assert process(event(j))=='ready'
|
||||
report['retry']={'transitions':transitions,'seconds':round(time.monotonic()-started,3),'replayed_attempt':fetch(a['id'])[1]['attempt']}
|
||||
report['invariants']['transient_backoff_exhaustion_then_replay']=True
|
||||
# Real corrupted JPEG whose header is valid: heavy decoder fails safely in worker.
|
||||
jpg=io.BytesIO();Image.new('RGB',(320,200),'blue').save(jpg,format='JPEG');truncated=jpg.getvalue()[:-100]
|
||||
a,j=upload(truncated,'image/jpeg');process(event(j));bad,_=fetch(a['id'])
|
||||
assert bad['state']=='failed' and bad['error_code']=='invalid_image'
|
||||
report['invariants']['invalid_decode_permanent_failed']=True
|
||||
# EXIF, alpha, small image and all on-disk variants checked by the real decoder.
|
||||
transforms=[]
|
||||
for kind in ['alpha','exif','small']:
|
||||
out=io.BytesIO()
|
||||
if kind=='alpha':
|
||||
Image.new('RGBA',(20,10),(255,0,0,0)).save(out,format='PNG');mime='image/png'
|
||||
elif kind=='exif':
|
||||
image=Image.new('RGB',(200,320),'green');exif=Image.Exif();exif[274]=6;image.save(out,format='JPEG',exif=exif);mime='image/jpeg'
|
||||
else:Image.new('RGB',(5,3),'yellow').save(out,format='PNG');mime='image/png'
|
||||
a,j=upload(out.getvalue(),mime);assert process(event(j))=='ready'
|
||||
r=c.call('GET','/api/v1/attachments/'+a['id']+'/content?variant=thumbnail',actor=bob)
|
||||
with Image.open(io.BytesIO(r['bytes'])) as thumb:
|
||||
expected=(20,10) if kind=='alpha' else (256,160) if kind=='exif' else (5,3)
|
||||
assert thumb.size==expected
|
||||
assert not thumb.getexif()
|
||||
if kind=='alpha':assert min(thumb.getpixel((0,0)))>=250
|
||||
transforms.append({'kind':kind,'size':thumb.size,'exif_removed':not bool(thumb.getexif())})
|
||||
report['transforms']=transforms
|
||||
pending,pending_job=upload()
|
||||
dry=cleanup(False);fixed=cleanup(True);repeat=cleanup(True)
|
||||
assert fetch(pending['id'])[0]['state']=='queued' and objects.head(pending_job['source_key']) is not None
|
||||
report['invariants']['cleanup_preserves_current_and_fresh_failed']=True
|
||||
assert repeat['counts']['ready_staging']==0
|
||||
report['invariants']['ready_staging_cleanup_idempotent']=True
|
||||
report['cleanup']=[dry,fixed,repeat]
|
||||
print(json.dumps(report,indent=2))
|
||||
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Figures from real accepted/ready and cgroup samples, no synthetic results."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
r=Path(__file__).resolve().parents[1]/'evidence'
|
||||
x=json.loads((r/'resources.json').read_text())['experiments']
|
||||
fig,axes=plt.subplots(1,2,figsize=(11,4.3),layout='constrained')
|
||||
labels=['320×200\n1,3 KiB','1800×1400\n7,21 MiB','10000×4000\n0,51 MiB']
|
||||
a=[max(v['accepted_seconds'] for v in e['replies']) for e in x]
|
||||
b=[max(v['ready_seconds'] for v in e['replies']) for e in x]
|
||||
axes[0].bar([i-.18 for i in range(3)],a,.36,label='p95 до 202')
|
||||
axes[0].bar([i+.18 for i in range(3)],b,.36,label='p95 до ready')
|
||||
axes[0].set(xticks=range(3),xticklabels=labels,ylabel='Секунды',title='10 одновременных загрузок · worker=1')
|
||||
axes[0].legend()
|
||||
peak=[int(e['after']['memory.peak'])/1048576 for e in x]
|
||||
anon=[max(int(dict(line.split() for line in s['worker']['memory.stat'].splitlines())['anon']) for s in e['samples'])/1048576 for e in x]
|
||||
axes[1].bar([i-.18 for i in range(3)],peak,.36,label='cgroup memory.peak')
|
||||
axes[1].bar([i+.18 for i in range(3)],anon,.36,label='Пик resident anon')
|
||||
axes[1].axhline(768,color='#d62728',linestyle='--',label='Лимит worker')
|
||||
axes[1].set(xticks=range(3),xticklabels=labels,ylabel='MiB',title='Worker + дочерний декодер')
|
||||
axes[1].legend(fontsize=8)
|
||||
for ax in axes:ax.grid(axis='y',alpha=.2);ax.set_axisbelow(True)
|
||||
fig.suptitle('ЛР 5 · реальные измерения · concurrency=1, prefetch=1')
|
||||
fig.savefig(r/'pipeline.png',dpi=150)
|
||||
fig.savefig(r/'pipeline.svg')
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Publication plot from measured cost/retention JSON; matplotlib is research-only."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
data=json.loads((ROOT/'evidence/observability-cost.json').read_text())
|
||||
modes=['off','full','sample10','sample25']
|
||||
labels=['Off','Full 100%','Sample 10%','Sample 25%']
|
||||
colors=['#64748b','#2563eb','#0d9488','#d97706']
|
||||
fig,axs=plt.subplots(2,3,figsize=(16,9),layout='constrained')
|
||||
def draw(ax,values,title,ylabel):
|
||||
med=[statistics.median(v) for v in values]
|
||||
low=[m-min(v) for m,v in zip(med,values)]
|
||||
high=[max(v)-m for m,v in zip(med,values)]
|
||||
ax.bar(range(4),med,color=colors,yerr=[low,high],capsize=4,alpha=.85)
|
||||
for i,v in enumerate(values):ax.scatter([i]*len(v),v,color='#0f172a',s=14,zorder=3)
|
||||
ax.set_xticks(range(4),labels,rotation=12)
|
||||
ax.set_title(title,loc='left',fontweight='bold')
|
||||
ax.set_ylabel(ylabel)
|
||||
ax.grid(axis='y',alpha=.2)
|
||||
ax.set_axisbelow(True)
|
||||
rows=[[r for r in data['runs'] if r['mode']==mode] for mode in modes]
|
||||
draw(axs[0,0],[[r['p95_seconds']*1000 for r in group] for group in rows],'HTTP p95','milliseconds')
|
||||
draw(axs[0,1],[[r['resource']['api']['cpu_seconds'] for r in group] for group in rows],'API CPU (both replicas)','CPU seconds / run + drain')
|
||||
draw(axs[0,2],[[r['collector_ingest']['otelcol_receiver_accepted_spans_total'] for r in group] for group in rows],'Accepted traces','spans / run + drain')
|
||||
draw(axs[1,0],[[r['resource']['otel-collector']['rx_bytes']/1024 for r in group] for group in rows],'Collector network received','KiB (includes scrape / protocol traffic)')
|
||||
draw(axs[1,1],[[r['resource']['api']['peak_sampled_anonymous_bytes']/1048576 for r in group] for group in rows],'API anonymous memory','MiB (sum of 1 Hz sampled replica peaks)')
|
||||
draw(axs[1,2],[[r['resource']['worker']['cpu_seconds'] for r in group] for group in rows],'Worker CPU incl renderer','CPU seconds / run + drain')
|
||||
fig.suptitle('Lab 6: cost of observability | 600 mixed requests per run (30 uploads) | 30 req/s | 3 repeats',fontsize=15,fontweight='bold')
|
||||
fig.savefig(ROOT/'evidence/observability-cost.png',dpi=150)
|
||||
fig.savefig(ROOT/'evidence/observability-cost.svg')
|
||||
print('Plot written from actual per-run JSON; dots = repeats, bars = median, whiskers = min/max.')
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Finite simultaneous jobs including 40 MP; record actual worker cgroup peak/RSS."""
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests'))
|
||||
from client import Client
|
||||
from images import png
|
||||
|
||||
|
||||
def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip()
|
||||
|
||||
|
||||
def memory(container):
|
||||
code="import json;from pathlib import Path;p=Path('/sys/fs/cgroup');print(json.dumps({k:(p/k).read_text().strip() for k in ['memory.current','memory.peak','memory.events','memory.stat','cpu.stat']}))"
|
||||
return json.loads(docker('exec',container,'python','-c',code))
|
||||
|
||||
|
||||
def main():
|
||||
c=Client(timeout=30);a,b,e,chat=c.bootstrap()
|
||||
profiles=[(320,200,False),(1800,1400,True),(10000,4000,False)]
|
||||
result=[]
|
||||
for w,h,noise in profiles:
|
||||
content=png(w,h,noise=noise)
|
||||
docker('compose','restart','worker');container=docker('compose','ps','-q','worker')
|
||||
before=memory(container);samples=[];done=threading.Event();start=time.monotonic()
|
||||
def sample():
|
||||
while not done.is_set():
|
||||
samples.append({'seconds':round(time.monotonic()-start,4),'worker':memory(container)})
|
||||
done.wait(.05)
|
||||
sampler=threading.Thread(target=sample);sampler.start()
|
||||
barrier=threading.Barrier(10)
|
||||
def upload(i):
|
||||
barrier.wait();t=time.monotonic()
|
||||
r=Client(timeout=30).call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=load.png',202,actor=a,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
accepted=time.monotonic()-t
|
||||
ready=Client(timeout=30).wait_ready(b,r['json'])
|
||||
assert ready['sha256']==hashlib.sha256(content).hexdigest()
|
||||
return {'index':i,'accepted_seconds':accepted,'ready_seconds':time.monotonic()-t,'status':ready['status']}
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: replies=list(pool.map(upload,range(10)))
|
||||
finally:done.set();sampler.join()
|
||||
after=memory(container);assert 'oom_kill 0' in after['memory.events']
|
||||
result.append({'width':w,'height':h,'bytes':len(content),'clients':10,'before':before,'after':after,'samples':samples,'replies':replies,'seconds':time.monotonic()-start})
|
||||
print(json.dumps({'method':'10 simultaneous uploads, worker concurrency=1; cgroup v2 samples include child renderer; anon is resident anonymous memory, memory.peak includes tmpfs/kernel/cache','experiments':result},indent=2))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Separate synchronized rare-error/slow-success retention experiment."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from observability_cost import set_mode, probes
|
||||
from observability import save, trace, wait
|
||||
import urllib.error
|
||||
|
||||
|
||||
def verify_presence(rows):
|
||||
for row in rows:
|
||||
if row["trace_sampled"]:
|
||||
data=wait(lambda:trace(row["trace_id"]),timeout=90)
|
||||
assert data.get("batches",data.get("resourceSpans"))
|
||||
row["tempo_trace_found"]=True
|
||||
else:
|
||||
try:
|
||||
trace(row["trace_id"])
|
||||
except urllib.error.HTTPError as error:
|
||||
assert error.code==404
|
||||
row["tempo_trace_found"]=False
|
||||
else:
|
||||
raise AssertionError("Unsampled trace unexpectedly stored")
|
||||
return rows
|
||||
|
||||
def main():
|
||||
result={'method':'Three repeats per full/10%/25% setting; each holds sessions lock until pg_locks confirms it. 3x503 plus one slow200 per repeat. Sampling is ordinary SDK head sampling without forced traceparent.','runs':[]}
|
||||
try:
|
||||
for mode,ratio in [('full',1),('sample10',.1),('sample25',.25)]:
|
||||
c=set_mode(1,ratio)
|
||||
actor=c.user('retention')
|
||||
for repeat in range(3):
|
||||
result['runs'].append({'mode':mode,'repeat':repeat,'probes':verify_presence(probes(c,actor,mode))})
|
||||
save('sampling-retention.json',result)
|
||||
print(mode,repeat,'retention recorded',flush=True)
|
||||
finally:set_mode(1,.25)
|
||||
print('PASS: synchronized rare 503/slow200 retention study; balanced sampling restored.')
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Private S3 and job envelopes do not expose source bytes or credentials."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
sys.path[:0]=['/app/src',str(Path.cwd()/'tests')]
|
||||
from client import Client
|
||||
from pipeline import connect,validate,event
|
||||
from objects import Objects
|
||||
from postgres import setting
|
||||
|
||||
c=Client();a,b,e,chat=c.bootstrap();att,raw=c.attachment(a,chat);ready=c.wait_ready(b,att)
|
||||
with connect() as conn:
|
||||
row=conn.execute('SELECT object_key, current_job_id FROM attachments WHERE 1=1 AND id = %s',(att['id'],)).fetchone()
|
||||
job=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(row['current_job_id'],)).fetchone()
|
||||
message=event(job);message['attempt']=max(1,job['attempt'])
|
||||
assert validate(message)
|
||||
required={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'}
|
||||
assert required <= set(message) <= required | {'traceparent','tracestate'}
|
||||
assert validate(message)
|
||||
assert len(json.dumps(message))<1024
|
||||
for path in ('/'+Objects(setting).bucket+'/'+row['object_key'],'/'+Objects(setting).bucket+'?list-type=2'):
|
||||
try:urllib.request.urlopen('http://s3:8333'+path,timeout=2);raise AssertionError('anonymous access')
|
||||
except urllib.error.HTTPError as error:assert error.code==403
|
||||
for key in ['schema_version','pipeline_version','attempt']:
|
||||
bad=dict(message);bad[key]=True;assert not validate(bad)
|
||||
for body in [dict(message,extra='x'),dict(message,created_at='bad'),dict(message,traceparent='bad')]:assert not validate(body)
|
||||
print(json.dumps({'anonymous_object_and_list_403':True,'job_fields':sorted(message),'job_bytes':len(json.dumps(message)),'no_source_bytes_or_credentials_in_envelope':True,'strict_invalid_envelope_checks':True},indent=2))
|
||||
@@ -0,0 +1,32 @@
|
||||
"""A continuously trickling client still reaches the absolute upload deadline."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import select
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import urlsplit
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client
|
||||
|
||||
c = Client()
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
target = urlsplit(c.base)
|
||||
connection = socket.create_connection((target.hostname, target.port), timeout=3)
|
||||
header = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=slow_' + uuid.uuid4().hex + '.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n'
|
||||
started = time.monotonic()
|
||||
connection.sendall(header.encode())
|
||||
response = b''
|
||||
while time.monotonic() - started < 20:
|
||||
if select.select([connection], [], [], .15)[0]:
|
||||
response += connection.recv(4096)
|
||||
if b'\r\n\r\n' in response:
|
||||
break
|
||||
connection.sendall(b'x')
|
||||
connection.close()
|
||||
elapsed = time.monotonic() - started
|
||||
assert response.startswith(b'HTTP/1.1 400') or response.startswith(b'HTTP/1.0 400'), response[:100]
|
||||
assert 14 <= elapsed < 19
|
||||
assert b'x-instance-id:' in response.lower() and b'x-request-id:' in response.lower()
|
||||
print(json.dumps({'trickle_seconds': round(elapsed, 3), 'http_status': 400, 'absolute_deadline_enforced': True}))
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Extra black-box tests, without changing the course's public tests."""
|
||||
import concurrent.futures
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
from urllib.parse import quote
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client as PublicClient, LAB
|
||||
|
||||
|
||||
class Client(PublicClient):
|
||||
def call(self, method, path, *args, headers=None, **kwargs):
|
||||
headers = dict(headers or {})
|
||||
if method == 'POST' and path.endswith('/messages'):
|
||||
headers.setdefault('Idempotency-Key', uuid.uuid4().hex)
|
||||
return super().call(method, path, *args, headers=headers, **kwargs)
|
||||
|
||||
|
||||
class APIContract(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.client = Client()
|
||||
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
|
||||
self.path = '/api/v1/chats/' + self.chat['id']
|
||||
|
||||
def test_malformed_json_and_extra_fields(self):
|
||||
for raw in (b'{', b'[]', b'{"text":"x","text":"y"}', b'{"text":NaN}', b'{"text":"\\ud800"}'):
|
||||
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'})
|
||||
for body in ({'text': 'ok', 'sender_id': self.eve['id']}, {'text': True}, {'text': ' '}, {'text': ''}, {'text': None}):
|
||||
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body=body)
|
||||
deep = b'{"text":' + b'[' * 1200 + b'0' + b']' * 1200 + b'}'
|
||||
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=deep, headers={'Content-Type': 'application/json'})
|
||||
|
||||
def test_unicode_limits(self):
|
||||
result = self.client.message(self.alice, self.chat, '👩' * 4000)['json']
|
||||
self.assertEqual(result['text'], '👩' * 4000)
|
||||
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': '👩' * 4001})
|
||||
|
||||
def test_unicode_nul_round_trip(self):
|
||||
import base64
|
||||
value = 'A\x00B\\u0000👩'
|
||||
registration = dict(self.alice['registration'], username='nul_' + uuid.uuid4().hex[:20], display_name=value)
|
||||
user = self.client.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json']
|
||||
self.assertEqual(user['display_name'], value)
|
||||
credential = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
|
||||
session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json']
|
||||
self.assertEqual(session['user'], user)
|
||||
actor = {'id': user['id'], 'token': session['token']}
|
||||
# Exercise the database path and then the Redis profile cache.
|
||||
for _ in range(2):
|
||||
self.assertEqual(self.client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'], user)
|
||||
chat = self.client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': value, 'member_ids': [actor['id']]}, schema='Chat')['json']
|
||||
self.assertEqual(chat['title'], value)
|
||||
self.assertEqual(self.client.call('GET', '/api/v1/chats/' + chat['id'], actor=actor, schema='Chat')['json'], chat)
|
||||
contents = [value, '\x00' * 4000, '👩' * 3999 + '\x00']
|
||||
for content in contents:
|
||||
self.assertEqual(self.client.message(actor, chat, content)['json']['text'], content)
|
||||
history = self.client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor, schema='MessagePage')['json']
|
||||
self.assertEqual([message['text'] for message in history['items']], contents)
|
||||
|
||||
def test_foreign_chat(self):
|
||||
for suffix in ('', '/messages'):
|
||||
self.client.call('GET', self.path + suffix, 404, actor=self.eve)
|
||||
self.client.call('POST', self.path + '/messages', 404, actor=self.eve, body={'text': 'hidden'})
|
||||
self.client.call('GET', '/api/v1/chats/' + str(uuid.uuid4()), 404, actor=self.alice)
|
||||
|
||||
def test_registration_race(self):
|
||||
body = dict(self.alice['registration'], username='race_' + uuid.uuid4().hex[:20])
|
||||
def register(_):
|
||||
return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status']
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
||||
statuses = list(executor.map(register, range(10)))
|
||||
self.assertEqual(statuses.count(201), 1)
|
||||
self.assertEqual(statuses.count(409), 9)
|
||||
|
||||
def test_invalid_chat_creation(self):
|
||||
members = ([self.alice['id'], self.alice['id']], [self.bob['id']], [self.alice['id'], str(uuid.uuid4())], [], [True])
|
||||
for member_ids in members:
|
||||
self.client.call('POST', '/api/v1/chats', 400, actor=self.alice, body={'title': 'rollback', 'member_ids': member_ids})
|
||||
|
||||
def test_cursor_scope_and_validation(self):
|
||||
for i in range(3):
|
||||
self.client.message(self.alice, self.chat, str(i))
|
||||
cursor = self.client.call('GET', self.path + '/messages?limit=1', actor=self.alice, schema='MessagePage')['json']['next_cursor']
|
||||
other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'other', 'member_ids': [self.alice['id']]}, schema='Chat')['json']
|
||||
self.client.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(cursor), 400, actor=self.alice)
|
||||
for query in ('limit=0', 'limit=101', 'limit=' + '9' * 5000, 'limit=true', 'limit=1&limit=2', 'cursor=', 'cursor=bad', 'cursor=e30=', 'offset=1'):
|
||||
self.client.call('GET', self.path + '/messages?' + query, 400, actor=self.alice)
|
||||
|
||||
def test_concurrent_messages_and_pagination(self):
|
||||
def send(i):
|
||||
return Client().message(self.alice if i % 2 else self.bob, self.chat, 'parallel-' + str(i))['json']
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
||||
sent = list(executor.map(send, range(100)))
|
||||
found = []
|
||||
cursor = None
|
||||
while True:
|
||||
url = self.path + '/messages?limit=7' + ('&cursor=' + quote(cursor) if cursor else '')
|
||||
page = self.client.call('GET', url, actor=self.alice, schema='MessagePage')['json']
|
||||
found.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
break
|
||||
self.assertEqual(len(found), 100)
|
||||
self.assertEqual(len({v['id'] for v in found}), 100)
|
||||
self.assertEqual({v['id'] for v in found}, {v['id'] for v in sent})
|
||||
self.assertEqual(found, sorted(found, key=lambda v: (v['created_at'], v['id'])))
|
||||
|
||||
def test_idempotency_concurrent_replicas(self):
|
||||
key = uuid.uuid4().hex
|
||||
def send(_):
|
||||
return Client().message(self.alice, self.chat, 'Один раз\x00👩', key=key)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
|
||||
replies = list(executor.map(send, range(32)))
|
||||
self.assertGreaterEqual(len({v['headers']['X-Instance-Id'] for v in replies}), 2)
|
||||
self.assertTrue(all(v['json'] == replies[0]['json'] for v in replies))
|
||||
page = self.client.call('GET', self.path + '/messages', actor=self.bob, schema='MessagePage')['json']
|
||||
self.assertEqual(page['items'], [replies[0]['json']])
|
||||
self.client.call('POST', self.path + '/messages', 409, actor=self.alice, body={'text': 'different'}, headers={'Idempotency-Key': key})
|
||||
self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=self.alice)
|
||||
self.client.call('POST', self.path + '/messages', 401, actor=self.alice, body={'text': 'Один раз\x00👩'}, headers={'Idempotency-Key': key})
|
||||
|
||||
def test_protocol_error_headers(self):
|
||||
self.client.call('TRACE', '/api/v1/users/me', 400, actor=self.alice)
|
||||
|
||||
def test_idempotency_scope(self):
|
||||
key = uuid.uuid4().hex
|
||||
one = self.client.message(self.alice, self.chat, 'same', key=key)['json']
|
||||
two = self.client.message(self.bob, self.chat, 'same', key=key)['json']
|
||||
other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'scope', 'member_ids': [self.alice['id']]}, schema='Chat')['json']
|
||||
three = self.client.message(self.alice, other, 'same', key=key)['json']
|
||||
self.assertEqual(len({one['id'], two['id'], three['id']}), 3)
|
||||
|
||||
def test_idempotency_semantic_body(self):
|
||||
key = uuid.uuid4().hex
|
||||
one = self.client.message(self.alice, self.chat, 'semantic', key=key)['json']
|
||||
two = self.client.call('POST', self.path + '/messages', 201, actor=self.alice, body={'attachment_ids': [], 'text': 'semantic'}, headers={'Idempotency-Key': key}, schema='Message')['json']
|
||||
self.assertEqual(one, two)
|
||||
|
||||
def test_idempotency_invalid_keys(self):
|
||||
for key in (None, '', 'x' * 129, 'space key', 'bad,key', 'slash/key'):
|
||||
headers = {} if key is None else {'Idempotency-Key': key}
|
||||
PublicClient().call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': 'valid'}, headers=headers)
|
||||
for raw in (b'{"text":true}', b'{"text":"ok","extra":1}', b'{"text":"ok","attachment_ids":false}'):
|
||||
self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'})
|
||||
|
||||
def test_idempotency_conflicting_race(self):
|
||||
key = uuid.uuid4().hex
|
||||
def send(i):
|
||||
return Client().call('POST', self.path + '/messages', [201, 409], actor=self.alice, body={'text': str(i % 2)}, headers={'Idempotency-Key': key}, schema=None)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
|
||||
replies = list(executor.map(send, range(32)))
|
||||
self.assertEqual(sum(v['status'] == 201 for v in replies), 16)
|
||||
self.assertEqual(sum(v['status'] == 409 for v in replies), 16)
|
||||
page = self.client.call('GET', self.path + '/messages', actor=self.alice, schema='MessagePage')['json']
|
||||
self.assertEqual(len(page['items']), 1)
|
||||
|
||||
@unittest.skipUnless(LAB >= 2, 'sessions start in lab2')
|
||||
def test_unicode_password_basic_and_logout(self):
|
||||
import base64
|
||||
registration = {'username': 'unicode_' + uuid.uuid4().hex[:16], 'display_name': 'Софья', 'password': '🔐' * 127 + ':'}
|
||||
self.client.call('POST', '/api/v1/users', 201, body=registration)
|
||||
encoded = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()
|
||||
headers = {'Authorization': 'Basic ' + encoded}
|
||||
session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers=headers, schema='Session')['json']
|
||||
actor = {'id': session['user']['id'], 'token': session['token']}
|
||||
self.client.call('GET', '/api/v1/users/me', actor=actor)
|
||||
self.client.call('GET', '/api/v1/users/me', 401, headers=headers)
|
||||
for authorization in ('Basic !!!', 'Bearer wrong', '', 'Basic Zm9v'):
|
||||
self.client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': authorization})
|
||||
self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
|
||||
self.client.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Independent tests for async image metadata, transformations and access."""
|
||||
import io
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'tests')]
|
||||
from client import Client
|
||||
from images import bomb
|
||||
|
||||
|
||||
class Images(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.c=Client(); self.alice,self.bob,self.eve,self.chat=self.c.bootstrap()
|
||||
self.path='/api/v1/chats/'+self.chat['id']
|
||||
self.raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
def test_async_variants_and_replay(self):
|
||||
queued,_=self.c.attachment(self.alice,self.chat)
|
||||
self.assertEqual(queued['status'],'queued')
|
||||
key=uuid.uuid4().hex
|
||||
one=self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json']
|
||||
self.assertEqual(self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json'],one)
|
||||
ready=self.c.wait_ready(self.alice,queued)
|
||||
self.assertEqual({v['name'] for v in ready['variants']},{'thumbnail','crop','optimized'})
|
||||
dimensions={v['name']:(v['width'],v['height']) for v in ready['variants']}
|
||||
self.assertEqual(dimensions,{'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)})
|
||||
for v in ready['variants']:
|
||||
import hashlib
|
||||
r=self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant='+v['name'],actor=self.bob)
|
||||
self.assertEqual(hashlib.sha256(r['bytes']).hexdigest(),v['sha256'])
|
||||
self.assertEqual(len(r['bytes']),v['size_bytes'])
|
||||
self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant=absent',404,actor=self.bob)
|
||||
|
||||
def test_mime_bytes_pixels_and_filename(self):
|
||||
path=self.path+'/attachments?filename=test.png'
|
||||
for raw,mime,status in [(b'not image','image/png',415),(self.raw[:20],'image/png',415),(self.raw,'image/jpeg',415),(bomb(),'image/png',413),(b'x'*(10485760+1),'image/png',413)]:
|
||||
self.c.call('POST',path,status,actor=self.alice,raw=raw,headers={'Content-Type':mime})
|
||||
for name in ('../x.png','a\\x.png','a\0.png','a\n.png','a'*256,''):
|
||||
self.c.call('POST',self.path+'/attachments?filename='+quote(name,safe=''),400,actor=self.alice,raw=self.raw,headers={'Content-Type':'image/png'})
|
||||
|
||||
def test_private_and_cross_chat(self):
|
||||
att,_=self.c.attachment(self.alice,self.chat)
|
||||
for suffix in ('','/content'):
|
||||
self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=self.eve)
|
||||
self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,401)
|
||||
other=self.c.call('POST','/api/v1/chats',201,actor=self.alice,body={'title':'other','member_ids':[self.alice['id']]},schema='Chat')['json']
|
||||
self.c.call('POST','/api/v1/chats/'+other['id']+'/messages',404,actor=self.alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
|
||||
|
||||
if __name__=='__main__': unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user