Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e636d4cc76 |
+23
-25
@@ -1,30 +1,28 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# Copy to .env and REPLACE all sample secrets with random unique values.
|
||||
# For local synthetic data only. Never commit .env.
|
||||
POSTGRES_DB=messenger
|
||||
POSTGRES_USER=lab
|
||||
POSTGRES_PASSWORD=REPLACE_WITH_RANDOM_LOCAL_PASSWORD
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
DB_POOL_MAX=8
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
S3_ADMIN_ACCESS_KEY=adminlab4
|
||||
S3_ADMIN_SECRET_KEY=REPLACE_WITH_DIFFERENT_RANDOM_ADMIN_SECRET
|
||||
S3_ACCESS_KEY=messengerapi
|
||||
S3_SECRET_KEY=REPLACE_WITH_DIFFERENT_RANDOM_API_SECRET
|
||||
|
||||
# RabbitMQ persistent queue (different from session Redis).
|
||||
BROKER_USER=labworker
|
||||
BROKER_PASSWORD=REPLACE_WITH_RANDOM_LOCAL_BROKER_PASSWORD
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
JOB_BACKLOG_LIMIT=1000
|
||||
WORKER_PREFETCH=1
|
||||
# Only for deliberate crash experiment: once. Keep off for normal operation.
|
||||
LAB5_FAULT_AFTER_S3=off
|
||||
|
||||
# Observability (loopback-only Grafana / diagnostics). No public OTLP ports.
|
||||
GRAFANA_ADMIN_PASSWORD=REPLACE_WITH_DIFFERENT_RANDOM_GRAFANA_PASSWORD
|
||||
OTEL_ENABLED=yes
|
||||
LAB6_FAULT_RETRY_ONCE=off
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
FROM python:3.12.10-slim-bookworm
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||
WORKDIR /srv
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt && groupadd --gid 999 app && useradd -m --uid 999 --gid app app
|
||||
COPY app ./app
|
||||
COPY db ./db
|
||||
USER app
|
||||
EXPOSE 8080
|
||||
CMD ["python", "-m", "app"]
|
||||
@@ -1,4 +1,4 @@
|
||||
PYTHON ?= python3
|
||||
PYTHON ?= python
|
||||
BASE_URL ?= http://localhost:8080
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
|
||||
@@ -1,37 +1,208 @@
|
||||
# Отчёт — лабораторная 6
|
||||
# Отчёт — лабораторная 6. Наблюдаемость
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор / группа: Авзалов Артур Марсович / БПИ2301.
|
||||
- Целевая оценка: 4; трек 5 не заявлен.
|
||||
- Статус: кандидат на 4. Реализация проверена на Windows/Docker Desktop 2026-11-17: все шесть сценариев наблюдаемости, публичный smoke, ряд регрессионных тестов Lab5 и визуализация Grafana прошли. Целевая оценка 4; окончательную оценку определяет преподаватель.
|
||||
- Commit/PR: указывается после публикации.
|
||||
- ИИ: ChatGPT помог проектировать, реализовывать инструментацию, инфраструктуру, тесты и документацию. Результаты ниже получены при запуске именно Lab6; ChatGPT помог исправить диагностический тест backend-down.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Docker Desktop (Linux containers, Compose V2), Python 3.9+ для публичных HTTP-тестов; Python 3.12 рекомендуется для локальных SDK/unit-проверок. Приложение использует Python 3.12.10. Желательно выделить Docker 6–8 GiB RAM с запасом. Перед запуском остановить Lab5 и другие стенды, занимающие 8080/3000/9090/3100/3200. Не удалять их volumes.
|
||||
|
||||
## Архитектура и решения
|
||||
PowerShell (корень проекта):
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```powershell
|
||||
Copy-Item .env.example .env
|
||||
notepad .env
|
||||
$env:PYTHONUTF8 = "1"
|
||||
# Заменить ВСЕ REPLACE_WITH_... на разные случайные значения.
|
||||
# Для паролей в DSN выбрать hex/URL-safe значения, не вставлять секреты в отчёт.
|
||||
# OTEL_ENABLED=yes, LAB5_FAULT_AFTER_S3=off, LAB6_FAULT_RETRY_ONCE=off.
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env run --rm --no-deps otel-collector validate --config=/etc/otelcol/config.yaml
|
||||
docker compose --env-file .env run --rm --no-deps --entrypoint /bin/promtool prometheus check config /etc/prometheus/prometheus.yml
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
docker compose --env-file .env ps
|
||||
python scripts/wait_ready.py
|
||||
make check
|
||||
make test
|
||||
```
|
||||
|
||||
`--wait` для backend без healthcheck подтверждает только запущенный контейнер, не готовность ingestion. Дождаться `/-/ready` Prometheus, `/ready` Loki/Tempo, `/api/health` Grafana, `up` для Collector; расширенные тесты делают это автоматически. Grafana: http://127.0.0.1:3000, пользователь `admin`, пароль `GRAFANA_ADMIN_PASSWORD` из локального `.env` (задаётся при первом создании volume). Dashboard MTUCI → Lab6 | Messenger observability (`/d/lab6-messenger/`). Prometheus alerts: http://127.0.0.1:9090/alerts. Порты диагностики доступны только на loopback; никакого наружного OTLP или прямых API-реплик.
|
||||
|
||||
Для локальных unit/SDK-проверок в отдельном Python 3.12 venv:
|
||||
|
||||
```powershell
|
||||
python -m pip install -r requirements.txt
|
||||
python scripts/check_lab6.py
|
||||
python tests/jobs_sql_contract.py
|
||||
python tests/own_unit.py
|
||||
python tests/telemetry_unit.py
|
||||
# Необязательная полная OpenAPI-валидация (отдельный dev requirement):
|
||||
python -m pip install -r requirements-dev.txt
|
||||
make validate
|
||||
```
|
||||
|
||||
Безопасная остановка: `docker compose --env-file .env down` (named volumes сохраняются). Только намеренный сброс СВОЕГО учебного стенда: `docker compose --env-file .env down -v` (удаляет и бизнес-данные, и историю телеметрии, не использовать для тестов). `.env`, реальные credentials и данные вне Git. Инициализация `migrate` транзакционна и идемпотентна; `s3-policy` готовит приватные credentials/policy, `s3-bootstrap` bucket, `telemetry-init` устанавливает права директорий named volumes. RabbitMQ queue создаётся приложением с durable/publisher confirms. Provisioning Grafana полностью из Git.
|
||||
|
||||
## Архитектура и перенос Lab5
|
||||
|
||||
```text
|
||||
HTTP -> HAProxy -> API1/API2 -> PostgreSQL + Redis + SeaweedFS
|
||||
-> outbox -> dispatcher -> RabbitMQ -> worker
|
||||
worker -> S3 + DB ready
|
||||
API / dispatcher / worker (включая дочерний image process)
|
||||
| OTLP HTTP/protobuf :4318, внутри telemetry
|
||||
v
|
||||
OTel Collector
|
||||
| metrics /metrics :8889 <- Prometheus scrape
|
||||
| logs native OTLP HTTP -> Loki
|
||||
| traces OTLP gRPC -> Tempo
|
||||
v
|
||||
Grafana (запросы к трём независимым stores)
|
||||
```
|
||||
|
||||
Бизнес-контракт Lab6 и исходные публичные тесты не изменены; добавляется необязательный диагностический response header `X-Trace-Id` на business HTTP при активной телеметрии. Метаданные, состояние job/outbox и идемпотентность живут в PostgreSQL; сессии в Redis AOF; оригиналы/варианты в S3; RabbitMQ доставляет задачи at-least-once. SDK и Collector не участвуют в транзакциях приложения. Worker сохраняет исходный timeout/lease и отдельный process для Pillow; таймаут/ошибки приводят к durable retry через Outbox. Изменения в `app/jobs.py`, `app/main.py`, `app/storage.py`, `app/attachments.py`, `app/image_worker.py` касаются инструментации, измерения backlog и обработки DB statement timeout; dispatcher обслуживает heartbeat даже в idle. Миграции 001–004 сохранены. Новых бизнес-таблиц не нужно: контракт job допускает optional `traceparent`, `tracestate`, они сохраняются в JSONB outbox.
|
||||
|
||||
### Роли / версии / данные / доступ
|
||||
|
||||
| Service | Версия (образ) | Данные, сеть, readiness |
|
||||
| --- | --- | --- |
|
||||
| api1/api2 | Python 3.12.10-slim-bookworm, FastAPI 0.115.12 | stateless, frontend/storage/telemetry; `/health/ready`, не опубликованы напрямую |
|
||||
| lb | HAProxy 3.1.7-alpine | frontend, 127.0.0.1:8080; readiness upstream |
|
||||
| postgres | 16.10-bookworm | postgres_data, storage, pg_isready |
|
||||
| redis | 7.4.5-alpine | redis_data, storage, redis-cli ping, AOF |
|
||||
| s3 | SeaweedFS 4.53 | s3_data, s3_config, storage; bootstrap + API head_bucket |
|
||||
| broker | RabbitMQ 4.1.8-management-alpine | rabbitmq_data, storage, diagnostics ping |
|
||||
| dispatcher/worker | Python 3.12.10, Pika 1.3.2, Pillow 11.3.0 | storage/telemetry, restart policy; live fault tests |
|
||||
| otel-collector | contrib 0.123.0 | memory-only bounded batches, telemetry; scrape :8888/:8889 |
|
||||
| prometheus | v3.2.1 | prometheus_data, telemetry, loopback 9090; /-/ready |
|
||||
| loki | 3.4.2 | loki_data, TSDB v13 filesystem, loopback 3100; /ready |
|
||||
| tempo | 2.7.2 | tempo_data, local blocks + WAL, loopback 3200; /ready |
|
||||
| grafana | 11.6.0 | grafana_data + Git provisioning, loopback 3000; /api/health |
|
||||
| telemetry-init | BusyBox 1.37.0 | one-shot chown только корней named volumes, сеть отсутствует |
|
||||
|
||||
API 384 MiB/0.75 CPU на реплику, worker 768 MiB/1 CPU, dispatcher 192 MiB/0.5 CPU, Collector 256 MiB/0.75 CPU, Prometheus 384 MiB, Loki/Tempo 512 MiB каждый, Grafana 384 MiB. Реальные CPU/RAM замеряются на стенде; лимиты не измерения. Prometheus retention 72h/512MB, Loki и Tempo retention 72h (асинхронная очистка backend, не жёсткий disk quota). Все конфиги `infra/otel/` монтируются read-only. `.env.example` содержит логические настройки runtime, Compose отображает POSTGRES_* в DATABASE_URL, Redis URL фиксируется внутренней сетью, S3_* в credentials, BROKER_* в AMQP credentials. Локальный файл `.env` не коммитится; секреты хранятся в окружении контейнеров учебного стенда, не production secret manager. `STAGING_RETENTION_SECONDS` при отсутствии настройки приложения равен 86400. `WORKER_PREFETCH=1`, один активный job на worker. `OTEL_ENABLED=yes` включает SDK; endpoint `http://otel-collector:4318` задаётся Compose. `GRAFANA_ADMIN_PASSWORD` только для локального интерфейса. `LAB6_FAULT_RETRY_ONCE` — операторская настройка, по умолчанию off (не API).
|
||||
|
||||
## Сигналы, запросы и единицы
|
||||
|
||||
SDK OTel API/SDK/OTLP HTTP 1.42.1, явная инструментация `app/telemetry.py`. Стабильные локальные names и ограниченные semantic attributes вместо автоматического capture. `service.name=messenger-api|messenger-dispatcher|messenger-worker`, `service.version=1.6.0`, `service.instance.id=api1|api2|container hostname`, `deployment.environment.name=lab`; Prometheus exporter переводит точки в подчёркивания, добавляет суффиксы секунд/байт/total. Все histogram buckets одинаковы на репликах: 0.005…600 s. Health измеряется, но исключён из business SLI и трассировки (чтобы probe не загрязнял трассы). HTTP duration измеряется до получения заголовков ответа; потоковая передача content целиком не измеряется. Не именуем p99 средним.
|
||||
|
||||
| Сигнал / Prometheus имя | Выражение / единица |
|
||||
| --- | --- |
|
||||
| HTTP throughput и 5xx | `sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))`, фильтр `status_class="5xx"` для ошибок; req/s |
|
||||
| p50/p95/p99 | `histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))`, аналогично 0.5/0.99; секунды |
|
||||
| Job outcomes | `sum by(outcome)(rate(lab_jobs_total[30s]))`, attempts/s; ready/retry/failed/stale/requeue; hard kill до метрики не считается завершённым attempt |
|
||||
| Worker processing | `histogram_quantile(0.95,sum by(le)(rate(lab_job_duration_seconds_bucket[30s])))`, секунды claimed attempt |
|
||||
| Time to ready | `histogram_quantile(0.95,sum by(le)(rate(lab_ready_duration_seconds_bucket[30s])))`, секунды от `processing_jobs.created_at` (время транзакции durable acceptance, близкое, но не точно равное моменту COMMIT) до commit ready, включает retry/backlog |
|
||||
| Durable backlog | `max(lab_queue_depth)`: число jobs в статусе queued, включая неопубликованные и delayed retry |
|
||||
| Broker queue | `max(lab_broker_ready)`: ready messages RabbitMQ через passive declare; `lab_broker_consumers` — число consumers; не включает unacked и не равно DB backlog |
|
||||
| Oldest age | `max(lab_queue_oldest_age_seconds)`: now−created_at старейшей queued job; секунды; нет queued = 0 |
|
||||
| DB pool | `lab_db_pool_used`, `lab_db_pool_waiting`, `lab_db_pool_capacity` по service_instance_id; connections/requests; used = size−available (включая reserved/создаваемые соединения) |
|
||||
| Worker saturation | `sum(lab_worker_busy)/clamp_min(sum(lab_worker_capacity),1)`; ratio, active/capacity. Один job на процесс. При отсутствии worker ряд исчезает по expiration, это не доказательство idle |
|
||||
| CPU/RAM | `lab_container_cpu_cores` (CPU seconds / wall seconds), `lab_container_memory_bytes` (cgroup v2 usage, fallback v1 CPU / RSS дерева процессов) |
|
||||
| Collector internal | `otelcol_exporter_queue_size`, `otelcol_exporter_queue_capacity`, `otelcol_exporter_send_failed_spans[_total]`, `otelcol_exporter_send_failed_log_records[_total]`, `otelcol_process_memory_rss` / `otelcol_process_memory_rss_bytes`; учитывать текущую нормализацию выбранной версии |
|
||||
|
||||
`method`, route template, `status_class`, `job_type`, `outcome`, `queue` — ограниченный набор label values, `service_instance_id` ограничен числом экземпляров за retention; нет raw URL, filename, user/chat/message/attachment/request/job ID. Метрики cumulative, Collector выставляет Prometheus exposition. `metric_expiration: 15s` ограничивает stale gauges (нет данных при недоступной БД, а не искусственный 0). Percentiles строятся по сумме histogram buckets всех реплик (не усреднение p95). При отсутствии событий rate/quantile может быть NaN: требуется свежая нагрузка, NaN не заменяем нулём искусственно.
|
||||
|
||||
### Сквозная трасса и логи
|
||||
|
||||
1. API создаёт реальный SERVER span на business route; `db.transaction` и `s3.upload_fileobj` видны как dependency spans. Request ID остаётся прежним; `http.completed` содержит request_id, status_code, method, route, trace/span ID, timestamp и severity.
|
||||
2. Во время транзакции job/outbox `job_payload()` добавляет W3C `traceparent`/`tracestate`. Dispatcher извлекает их; создаёт PRODUCER span и внедряет **свой** context в сообщение RabbitMQ. Outbox хранит исходный контекст. На retry вызов из CONSUMER сохраняет его context; lease rescue/replay без активного span использует первый сохранённый context.
|
||||
3. Worker извлекает context, CONSUMER является child producer; проверяет job/lease в БД. `image.subprocess` передаёт context дочернему процессу в env без секретов телеметрии; видны `image.transform`, `db.load_staging_metadata`, `s3.get_object`, `s3.put_object`, `s3.copy_object`, затем `db.publish_result`. Worker завершает `job.ready` или `job.retry`/`job.failed`, ack после durable результата/retry. В случае hard kill последний span может не успеть экспортироваться; retry продолжает trace.
|
||||
4. В Grafana Explore → Tempo по `X-Trace-Id` / trace_id; открываем span; Loki: `{service_name="messenger-api"} | trace_id="<trace_id>" | request_id="<request_id>"`, worker `{service_name="messenger-worker"} | trace_id="<trace_id>"`. Поля `trace_id`, `span_id`, `event` — OTLP structured metadata; тела записей фиксированные технические события. Технический job_id только в trace/log для конкретного учебного задания; SQL text, exception messages/stack traces, raw URLs, Authorization, passwords, session tokens, тело сообщений и файлы не передаются SDK. Отключён uvicorn access log, только отдельный безопасный logger передаётся в OTLP. Это проверкой синтетических sentinel полей подтверждено локально, не исчерпывающий security-аудит.
|
||||
|
||||
### SLI / SLO и alerts
|
||||
|
||||
SLI = 1−(число business HTTP 5xx / число всех завершённых business HTTP responses), health исключён, 4xx клиента не считаются отказом сервера. Окно 5 минут; **SLO >= 99%** (учебный, выбран нами, не обещание SLA). Dashboard SLI: `1 - ((sum(increase(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[5m])) or vector(0)) / sum(increase(lab_http_requests_total{route=~"/api/.*"}[5m])))`. Если нет трафика — нет измеримого SLI. SLI не учитывает отказ между клиентом и HAProxy либо ответы самого HAProxy, поскольку SDK в API; для production нужны LB/probe metrics. Время до ready отдельно измеряется histogram; успешные ready не учитывают навсегда failed, поэтому не подменяем им SLI доступности.
|
||||
|
||||
Правила `infra/otel/prometheus/alerts.yml`: `LabAPIHighErrorRate` >5% business 5xx за 30 s, `for: 9s`; `LabQueueLag` age >15 s и depth>0, `for: 12s`. Это локальные diagnostic alert thresholds, не burn-rate track 5. Prometheus evaluation/scrape 3s; firing/pending/inactive (inactive после firing = resolved) в Prometheus UI/API, Grafana dashboard `ALERTS`. Проверка искусственными отказами — ниже; внешней доставки нет.
|
||||
|
||||
### Backpressure и сбой телеметрии
|
||||
|
||||
SDK async BatchSpan/BatchLogRecordProcessor queue 512 items/64 batch, HTTP exporter 2s timeout; metric reader 3s interval/2s export timeout. Не ждём backend для HTTP/normal commit; дочерний process ограниченно flush'ит (добавляет latency, остаётся внутри JOB_TIMEOUT), потеря телеметрии допустима. Collector memory_limiter 180 MiB (spike 40 MiB), container limit 256 MiB, GOMEMLIMIT 180MiB. Batch 128/256 items, timeout 1s; каждый Tempo/Loki exporter queue 64 batches, один consumer, request timeout 3s, retry до 10s. При переполнении/отказе telemetry может быть потеряна; нет бесконечной durable очереди и нет гарантий delivery telemetry. Prometheus scrape хранит history самостоятельно. `backend-down` проверяет успешные бизнес-записи/обработку изображения при остановленном Tempo, рост failed exports, sampled RSS и queue size, отсутствие OOM/restart и свежий trace после восстановления. Failed export counter отражает неуспешный экспорт Collector (не точное число всех потерянных spans SDK; повторные попытки и обрывы могут влиять). Измерение на коротком сценарии не доказательство бессрочной нагрузки.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Локальные проверки (в среде подготовки, 2026-11-17) см. [evidence/offline_validation.md](evidence/offline_validation.md). Фактические результаты Docker-прогона на Windows 2026-11-17 сведены в [evidence/verified_results_2026-11-17.md](evidence/verified_results_2026-11-17.md). JSON-файлы сформированы тестами на стенде; скриншоты Grafana — в `evidence/grafana/`. Неповторённые проверки отмечены отдельно. Скрипты добавлены отдельно, исходные `tests/smoke.py`, `client.py`, `load.py`, `resilience.py`, fixtures и OpenAPI сохранены byte-for-byte.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий | Команда/сценарий | Результат / evidence |
|
||||
| --- | --- | --- |
|
||||
| Структура, SQL, SDK | `make check`, `python scripts/check_lab6.py`, `python tests/jobs_sql_contract.py`, `python tests/telemetry_unit.py`, `python tests/own_unit.py` | локально PASS; SDK wire-format test не проверяет Collector/backend |
|
||||
| Основной API и регрессия Lab5 | `make test`, `attachments_negative`, `idempotency_parallel`, `own_unit`, `pipeline_resilience` (paused/outbox-crash/worker-crash) | PASS на Lab6; отдельные `resilience.py` и `pipeline_extended.py` в данном прогоне не подтверждены |
|
||||
| Все три stores, dashboard/datasources, метрики + низкая кардинальность | `observability_live.py --scenario metrics` | PASS: ожидаемые ряды и отсутствие запрещённых ID labels; `evidence/lab6-metrics.json` |
|
||||
| Upload/producer/consumer/retry, лог+trace, S3+DB result | `--scenario trace-retry` | PASS: API/producer/consumer/S3/DB с retry, связанные логи; `evidence/lab6-trace-retry.json` |
|
||||
| Queue age / depth / SLO & alert | `--scenario queue-lag` | PASS: возраст 27.574387 s; queued=1; firing → ready → resolved; `evidence/lab6-queue-lag.json` |
|
||||
| DB latency / 5xx alert | `--scenario db-error` | PASS: QueryCanceled/503, error ratio 1.0; firing → resolved; `evidence/lab6-db-error.json` |
|
||||
| Collector ограничен, telemetry backend down | `--scenario backend-down` | PASS после исправления теста: 16 успешных записей, sampled max RSS 200.43 MiB / 256 MiB, queue 18/64, свежая трасса после восстановления; `evidence/lab6-backend-down.json` |
|
||||
| Сохранение метрик/логов/traces при restart | `--scenario history` | PASS: сохранены метрики, логи, трассы после перезапуска backend; `evidence/lab6-history.json` |
|
||||
| Нагрузка, percentiles, ресурсы | `tests/load.py`, `attachments_concurrent.py`, dashboard CPU/RAM, PromQL | PASS: 10/10 HTTP 202 → ready, по 3 варианта; acceptance 3.976 RPS, p50 2296.5 ms, max 2484.0 ms, все ready за 36.406 s; `evidence/attachments-concurrent.json`; 16 панелей Grafana показывают данные. Отдельный `tests/load.py` ещё не подтверждён |
|
||||
|
||||
## Эксперименты и отказы
|
||||
### Команды полного прогона на Windows
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Список включает как уже выполненные, так и дополнительные регрессионные проверки. Фактический статус — в сводке доказательств; команды сами по себе не означают PASS.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
```powershell
|
||||
$env:PYTHONUTF8 = "1"
|
||||
$env:LAB5_ALLOW_FAILURE_TEST = "yes"
|
||||
$env:LAB6_ALLOW_FAILURE_TEST = "yes"
|
||||
# Все fault tests строго последовательно, только на отдельном учебном стенде.
|
||||
make test
|
||||
python tests/attachments_negative.py
|
||||
python tests/idempotency_parallel.py
|
||||
python tests/own_unit.py
|
||||
python tests/pipeline_resilience.py --scenario paused
|
||||
python tests/pipeline_resilience.py --scenario outbox-crash
|
||||
python tests/pipeline_resilience.py --scenario worker-crash
|
||||
python tests/pipeline_extended.py --scenario broker-persistence
|
||||
python tests/pipeline_extended.py --scenario failed-replay
|
||||
python tests/pipeline_extended.py --scenario staging-cleanup
|
||||
python tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
|
||||
python tests/attachments_concurrent.py --concurrency 10 --output evidence/attachments-concurrent.json
|
||||
python tests/observability_live.py --scenario metrics
|
||||
python tests/observability_live.py --scenario trace-retry
|
||||
python tests/observability_live.py --scenario queue-lag
|
||||
python tests/observability_live.py --scenario db-error
|
||||
python tests/observability_live.py --scenario backend-down
|
||||
python tests/observability_live.py --scenario history
|
||||
python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||
python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
|
||||
docker compose --env-file .env up -d --no-deps --wait api1 api2
|
||||
make test
|
||||
```
|
||||
|
||||
На Linux соответствующие hooks `restart.sh`, `stop-one.sh` (executable). Не запускать сценарии одновременно. Fault scripts управляют реальными контейнерами, не удаляют volumes. `trace-retry` временно пересоздаёт worker с operator-only fault once, затем восстанавливает off. `db-error` удерживает lock не более 60s после захвата; освобождение в конце транзакции; не применять на production. `history` перезапускает telemetry stores с named volumes. При ошибке смотреть `docker compose logs --tail=80 <service>`, не публиковать secrets. `make check` не заменяет runtime. Если Docker не стартует, не менять публичные тесты ради PASS.
|
||||
|
||||
## Инциденты и runbook
|
||||
|
||||
**Инцидент 1, гипотеза: остановлен worker.** `queue-lag` принимает синтетический upload при остановленном worker. Через метрики backlog/oldest age определяем растущую очередь, сравниваем broker ready/consumer count; API может продолжать отвечать 202, CPU worker отсутствует, DB pool нормальный. `LabQueueLag` firing; trace upload → publish есть, consume отсутствует. Запустить worker и дождаться ready, depth=0, alert inactive. Время обнаружения/возраст/глубина — в JSON результата. Факт (17.11): `queue-lag` PASS, age 27.574387 s, queued=1 при firing; после запуска worker задание стало ready, alert resolved. См. `evidence/lab6-queue-lag.json` и [графики очереди](evidence/grafana/02-worker-and-queue.png).
|
||||
|
||||
**Инцидент 2, гипотеза: DB заблокирована, не медленный handler.** `db-error` удерживает кратковременный ACCESS EXCLUSIVE LOCK на `users` в отдельной psql транзакции, health SELECT 1 остаётся рабочим. Создаётся несколько business GET `/api/v1/users/me`, которые получают 503 после database statement timeout ~5s или pool timeout. В Grafana: HTTP errors/p95 растут, `db.transaction` занимает основное время и имеет error.type QueryCanceled, связанный http.completed лог с request_id; отличить от долгого обработчика с нормальными dependency spans. `LabAPIHighErrorRate` firing; после COMMIT lock запросы снова 200, error rate падает и alert inactive. Факт (17.11): `db-error` PASS, QueryCanceled и HTTP 503 обнаружены, error ratio=1.0 при firing, после снятия блокировки alert resolved. См. `evidence/lab6-db-error.json` и [графики API](evidence/grafana/01-api-and-jobs.png).
|
||||
|
||||
**Инцидент 3: Tempo недоступна.** `backend-down` останавливает Tempo, посылает synthetic messages и image upload. Prometheus продолжает считать business telemetry; Collector failed spans/queue/RSS показывают сбой экспорта, API/worker продолжают обработку. После возврата Tempo создаётся новая трасса, старые могли потеряться. Факт (17.11): 16 успешных записей, обработка изображения сохранена; RSS Collector до отказа 197.35 MiB, наблюдаемый максимум 200.43 MiB при лимите 256 MiB; очередь выросла с 0/64 до 18/64. Ошибки экспорта наблюдались, свежая трасса после восстановления доступна. См. `evidence/lab6-backend-down.json` и [графики Collector](evidence/grafana/04-collector-alerts-slo.png). Первоначальный тест завершился на проверке RSS; исправлен только тест (поддержка варианта имени метрики с `_bytes`, строгая проверка очереди и диагностика лимитов); последующий живой прогон PASS.
|
||||
|
||||
Для нагрузки фиксировать hardware/выделенные Docker CPU/RAM, прогрев после `up`, дату, commit, размер файла 111817 B, длительность 30s и concurrency 4 для POST messages, concurrency 10 для upload; сохранить `load.json`, `attachments-concurrent.json`, показатели p50/p95/p99 из dashboard, CPU/RAM через PromQL/периодические замеры `docker stats`. Closed-loop нагрузки не доказывают максимальную capacity; p95 из histogram приблизителен. В `evidence/` хранить только обезличенные summary и технические IDs синтетических trace, не raw secrets/requests. Скриншоты Grafana/alerts полезны как дополнение, не вместо stored results.
|
||||
|
||||
### Визуальная проверка Grafana (17.11.2026)
|
||||
|
||||
Все 16 панелей на четырёх приложенных скриншотах содержат данные реального запуска. Снимки показывают часовой интервал с тестовыми инцидентами и последующей нагрузкой; точные значения и состояния берутся из результатов сценариев, визуальные оценки округлены.
|
||||
|
||||
| Доказательство | Что видно |
|
||||
| --- | --- |
|
||||
| [API и jobs](evidence/grafana/01-api-and-jobs.png) | Throughput, всплеск доли 5xx до 100%, p95/p99 около 10 s в момент сбоя, job outcomes |
|
||||
| [Worker и очередь](evidence/grafana/02-worker-and-queue.png) | Processing duration, time to ready (с учётом очереди), backlog, пик возраста около 27 s |
|
||||
| [Pool и ресурсы](evidence/grafana/03-pool-and-resources.png) | DB pool capacity 8 на реплику, worker saturation достигает 100%, CPU и sampled memory по экземплярам |
|
||||
| [Collector, alerts, SLO](evidence/grafana/04-collector-alerts-slo.png) | Очередь экспорта Tempo при ограничении 64, failed exports, firing/pending, падение SLI ниже 99% и последующее восстановление |
|
||||
|
||||
Для показа связи лога и трассы в Grafana Explore используйте сохранённый trace ID `4d8711b71965876a59aa5594ad0d51a0` из `trace-retry`, пока данные доступны по retention. При истечении retention повторите сценарий. PNG подтверждают визуализацию метрик; связанные логи и spans подтверждены тестом `trace-retry`, а не отдельными скриншотами Explore.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Один Docker-host и одиночные PG, Redis, S3, RabbitMQ, LB, Collector и telemetry backends. При их потере не заявляем HA. Бизнес-данные и telemetry разделены, at-least-once не exactly-once. Наблюдаемость best-effort: SDK queue/Collector memory pressure, hard crash, offline backend могут приводить к пропускам traces/logs/metrics. Трассируем зависимость до заголовков/вызова S3, не каждую SQL-команду и не весь downloaded stream. No sampling research / burn-rate track. Нагрузочные и fault evidence приведены выше. Замеры ограничены коротким сценарием и 10 одновременными upload; отдельно `tests/load.py`, повторный полный persistence/failover бизнес-стенда и три сценария `pipeline_extended.py` в предоставленном выводе Lab6 отсутствуют. Параметры CPU/RAM Docker-host и полный вывод JSON не предоставлены для этой сводки. Примерные пароли использовались на локальном тестовом стенде; при новом развёртывании заменить их перед запуском, не коммитить `.env`.
|
||||
|
||||
## Источники
|
||||
|
||||
- [Описание задания](README.md), [контракт observability](contracts/observability.md), [схема job](contracts/jobs.schema.json), [карта курса](COURSE.md).
|
||||
- [OpenTelemetry Collector configuration](https://opentelemetry.io/docs/collector/configuration/), [internal telemetry](https://opentelemetry.io/docs/collector/internal-telemetry/), [context propagation](https://opentelemetry.io/docs/concepts/context-propagation/), [W3C Trace Context](https://www.w3.org/TR/trace-context/).
|
||||
- [Prometheus histograms](https://prometheus.io/docs/practices/histograms/), [Loki OTLP ingestion](https://grafana.com/docs/loki/latest/send-data/otel/), [Tempo configuration](https://grafana.com/docs/tempo/latest/configuration/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/).
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Lab 3 messenger API."""
|
||||
@@ -0,0 +1,4 @@
|
||||
import os
|
||||
import uvicorn
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run("app.main:app", host="0.0.0.0", port=int(os.getenv("APP_PORT", "8080")), workers=1, access_log=False, timeout_graceful_shutdown=8)
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
"""Messenger HTTP handlers; state lives only in PostgreSQL/Redis."""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import psycopg
|
||||
from argon2 import PasswordHasher
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from .auth import current_user, hasher, login, logout, public_user
|
||||
from .core import APIError, invalid, iso, make_cursor, parse_limit, parse_uuid, read_cursor
|
||||
from .models import CreateChat, CreateMessage, CreateUser
|
||||
from .storage import db
|
||||
|
||||
router = APIRouter()
|
||||
KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$", re.ASCII)
|
||||
|
||||
|
||||
def actor(req: Request) -> dict:
|
||||
return getattr(req.state, "actor", None) or current_user(req)
|
||||
|
||||
|
||||
def chat_detail(conn, chat: dict) -> dict:
|
||||
members = conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position", (chat["id"],)).fetchall()
|
||||
return {"id": str(chat["id"]), "title": bytes(chat["title"]).decode("utf-8"), "created_at": iso(chat["created_at"]), "member_ids": [str(x["user_id"]) for x in members]}
|
||||
|
||||
|
||||
def message_detail(conn, row: dict) -> dict:
|
||||
linked = conn.execute("SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", (row["id"],)).fetchall()
|
||||
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": bytes(row["text"]).decode("utf-8"), "created_at": iso(row["created_at"]), "attachment_ids": [str(x["attachment_id"]) for x in linked]}
|
||||
|
||||
|
||||
def require_chat(conn, chat_id: str, actor_id: str) -> dict:
|
||||
row = conn.execute("SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id WHERE c.id=%s AND cm.user_id=%s", (chat_id, actor_id)).fetchone()
|
||||
if row is None:
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/health/ready")
|
||||
def ready():
|
||||
from .storage import ready as storage_ready
|
||||
if not storage_ready():
|
||||
raise APIError(503, "unavailable", "Service not ready")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.post("/api/v1/users", status_code=201)
|
||||
def register(body: CreateUser):
|
||||
hashed = hasher.hash(body.password)
|
||||
ident = str(uuid.uuid4())
|
||||
try:
|
||||
with db() as conn:
|
||||
row = conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (ident, body.username, body.display_name.encode("utf-8"), hashed)).fetchone()
|
||||
return public_user(row)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise APIError(409, "conflict", "Username already exists") from None
|
||||
|
||||
|
||||
@router.post("/api/v1/auth/sessions", status_code=201)
|
||||
def sessions(req: Request):
|
||||
return login(req)
|
||||
|
||||
|
||||
@router.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def sessions_delete(req: Request):
|
||||
logout(req)
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/api/v1/users/me")
|
||||
def users_me(req: Request):
|
||||
return actor(req)
|
||||
|
||||
|
||||
@router.post("/api/v1/chats", status_code=201)
|
||||
def chats_create(req: Request, body: CreateChat):
|
||||
ident = actor(req)["id"]
|
||||
if ident not in body.member_ids:
|
||||
raise invalid("Current user must belong to the chat")
|
||||
chat_id = str(uuid.uuid4())
|
||||
try:
|
||||
with db() as conn:
|
||||
with conn.transaction():
|
||||
# Validate all members; FK constraints are the final authority for races.
|
||||
found = conn.execute("SELECT id FROM users WHERE id = ANY(%s::uuid[])", (body.member_ids,)).fetchall()
|
||||
if len(found) != len(body.member_ids):
|
||||
raise invalid("Unknown member")
|
||||
row = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (chat_id, body.title.encode("utf-8"))).fetchone()
|
||||
for pos, member in enumerate(body.member_ids):
|
||||
conn.execute("INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)", (chat_id, member, pos))
|
||||
result = chat_detail(conn, row)
|
||||
return result
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise invalid("Unknown member") from None
|
||||
|
||||
|
||||
@router.get("/api/v1/chats")
|
||||
def chats_list(req: Request, limit: str | None = None, cursor: str | None = None):
|
||||
who = actor(req)["id"]
|
||||
n = parse_limit(limit)
|
||||
mark = read_cursor(cursor, "chats", who)
|
||||
with db() as conn:
|
||||
if mark:
|
||||
existing = conn.execute("SELECT 1 FROM chats c JOIN chat_members cm ON c.id=cm.chat_id WHERE c.id=%s AND c.created_at=%s AND cm.user_id=%s", (mark[1], mark[0], who)).fetchone()
|
||||
if not existing:
|
||||
raise invalid("Invalid cursor")
|
||||
rows = conn.execute("""
|
||||
SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id
|
||||
WHERE cm.user_id=%s AND (%s::timestamptz IS NULL OR (c.created_at,c.id)>(%s::timestamptz,%s::uuid))
|
||||
ORDER BY c.created_at,c.id LIMIT %s
|
||||
""", (who, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall()
|
||||
items = [chat_detail(conn, row) for row in rows[:n]]
|
||||
more = len(rows) > n
|
||||
last = rows[n-1] if more else None
|
||||
return {"items": items, "next_cursor": make_cursor("chats", who, last["created_at"], str(last["id"])) if last else None}
|
||||
|
||||
|
||||
@router.get("/api/v1/chats/{chat_id}")
|
||||
def chats_get(req: Request, chat_id: str):
|
||||
who = actor(req)["id"]
|
||||
ident = parse_uuid(chat_id)
|
||||
with db() as conn:
|
||||
row = require_chat(conn, ident, who)
|
||||
return chat_detail(conn, row)
|
||||
|
||||
|
||||
@router.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def messages_create(req: Request, chat_id: str, body: CreateMessage):
|
||||
who = actor(req)["id"]
|
||||
ident = parse_uuid(chat_id)
|
||||
key = req.headers.get("Idempotency-Key")
|
||||
if not key or not KEY_RE.fullmatch(key):
|
||||
raise invalid("Invalid Idempotency-Key")
|
||||
# Lab3 has only text; canonical JSON avoids unstable formatting/field ordering.
|
||||
fingerprint = hashlib.sha256(json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
|
||||
lock_scope = f"{who}:POST:/api/v1/chats/{ident}/messages:{key}"
|
||||
with db() as conn:
|
||||
with conn.transaction():
|
||||
require_chat(conn, ident, who)
|
||||
# Transaction-scoped advisory lock is GLOBAL across API replicas.
|
||||
conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (lock_scope,))
|
||||
previous = conn.execute("""
|
||||
SELECT idem.body_sha256, m.* FROM message_idempotency idem
|
||||
JOIN messages m ON m.id=idem.message_id
|
||||
WHERE idem.actor_id=%s AND idem.chat_id=%s AND idem.method='POST' AND idem.idem_key=%s
|
||||
""", (who, ident, key)).fetchone()
|
||||
if previous:
|
||||
if previous["body_sha256"] != fingerprint:
|
||||
raise APIError(409, "conflict", "Idempotency key used with another body")
|
||||
return message_detail(conn, previous)
|
||||
# 404 for foreign/nonexistent IDs; check BEFORE any message insert.
|
||||
if body.attachment_ids:
|
||||
owned = conn.execute("SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])", (ident, body.attachment_ids)).fetchall()
|
||||
if len(owned) != len(body.attachment_ids):
|
||||
raise APIError(404, "not_found", "Attachment not found")
|
||||
if any(item["status"] == "failed" for item in owned):
|
||||
raise APIError(409, "conflict", "Failed attachment cannot be linked")
|
||||
message_id = str(uuid.uuid4())
|
||||
row = conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (message_id, ident, who, body.text.encode("utf-8"))).fetchone()
|
||||
for pos, attachment_id in enumerate(body.attachment_ids):
|
||||
conn.execute("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", (message_id, attachment_id, pos))
|
||||
conn.execute("INSERT INTO message_idempotency(actor_id,chat_id,method,idem_key,body_sha256,message_id) VALUES (%s,%s,'POST',%s,%s,%s)", (who, ident, key, fingerprint, message_id))
|
||||
return message_detail(conn, row)
|
||||
|
||||
|
||||
@router.get("/api/v1/chats/{chat_id}/messages")
|
||||
def messages_list(req: Request, chat_id: str, limit: str | None = None, cursor: str | None = None):
|
||||
who = actor(req)["id"]
|
||||
ident = parse_uuid(chat_id)
|
||||
n = parse_limit(limit)
|
||||
mark = read_cursor(cursor, "messages", ident)
|
||||
with db() as conn:
|
||||
require_chat(conn, ident, who)
|
||||
if mark:
|
||||
existing = conn.execute("SELECT 1 FROM messages WHERE id=%s AND chat_id=%s AND created_at=%s", (mark[1], ident, mark[0])).fetchone()
|
||||
if not existing:
|
||||
raise invalid("Invalid cursor")
|
||||
rows = conn.execute("""
|
||||
SELECT * FROM messages WHERE chat_id=%s
|
||||
AND (%s::timestamptz IS NULL OR (created_at,id)>(%s::timestamptz,%s::uuid))
|
||||
ORDER BY created_at,id LIMIT %s
|
||||
""", (ident, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall()
|
||||
with db() as conn:
|
||||
items = [message_detail(conn, row) for row in rows[:n]]
|
||||
last = rows[n-1] if len(rows)>n else None
|
||||
return {"items": items, "next_cursor": make_cursor("messages", ident, last["created_at"], str(last["id"])) if last else None}
|
||||
@@ -0,0 +1,240 @@
|
||||
"""Lab5 attachments: synchronous durable admission, asynchronous image pipeline.
|
||||
|
||||
202 means verified bytes were staged to private S3 and metadata + a job + an
|
||||
outbox event were committed in ONE PostgreSQL transaction. No broker availability
|
||||
is required for admission; the dispatcher replays unsent intents.
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
from contextlib import closing
|
||||
|
||||
import boto3
|
||||
from boto3.s3.transfer import TransferConfig
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from PIL import Image, ImageFile, UnidentifiedImageError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .auth import current_user
|
||||
from .core import APIError, invalid, iso, parse_uuid
|
||||
from .storage import db
|
||||
from . import telemetry as ot
|
||||
|
||||
router = APIRouter()
|
||||
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
SPOOL_MEMORY_BYTES = 1024 * 1024
|
||||
S3_BUFFER_BYTES = 256 * 1024
|
||||
DECODE_LIMIT = threading.BoundedSemaphore(1)
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
|
||||
|
||||
def s3_client():
|
||||
return ot.S3Tracing(boto3.client(
|
||||
"s3", endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
|
||||
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
|
||||
region_name="us-east-1", use_ssl=False,
|
||||
config=Config(s3={"addressing_style": "path"}, connect_timeout=3,
|
||||
read_timeout=10, retries={"max_attempts": 1}),
|
||||
))
|
||||
|
||||
|
||||
def s3_error(exc):
|
||||
return APIError(503, "unavailable", "Object storage unavailable")
|
||||
|
||||
|
||||
def format_attachment(row, variants=()):
|
||||
return {
|
||||
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]), "filename": row["filename"],
|
||||
"content_type": row["content_type"], "size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"], "status": row["status"],
|
||||
"variants": [{k: item[k] for k in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
|
||||
for item in variants] if row["status"] == "ready" else [],
|
||||
"error_code": row["error_code"] if row["status"] == "failed" else None,
|
||||
"created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
def require_member(conn, chat_id, user_id):
|
||||
if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone():
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
|
||||
|
||||
def attachment_row(conn, attachment_id, user_id):
|
||||
row = conn.execute("""
|
||||
SELECT a.* FROM attachments a
|
||||
JOIN chat_members cm ON cm.chat_id=a.chat_id
|
||||
WHERE a.id=%s AND cm.user_id=%s
|
||||
""", (attachment_id, user_id)).fetchone()
|
||||
if not row:
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
return row
|
||||
|
||||
|
||||
def validate_filename(filename: str):
|
||||
if not (1 <= len(filename) <= 255) or any(c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename):
|
||||
raise invalid("Invalid filename")
|
||||
|
||||
|
||||
def validate_image(fileobj, mime: str):
|
||||
"""Validate MIME, magic, dimensions and complete decode; worker does the resize."""
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
with Image.open(fileobj) as im:
|
||||
if im.format not in ("PNG", "JPEG"):
|
||||
raise APIError(415, "unsupported_media_type", "Unsupported image encoding")
|
||||
actual = "image/png" if im.format == "PNG" else "image/jpeg"
|
||||
if actual != mime:
|
||||
raise APIError(415, "unsupported_media_type", "Content-Type does not match image")
|
||||
if im.width * im.height > MAX_PIXELS:
|
||||
raise APIError(413, "payload_too_large", "Image dimensions too large")
|
||||
im.verify()
|
||||
fileobj.seek(0)
|
||||
with DECODE_LIMIT:
|
||||
with Image.open(fileobj) as im:
|
||||
im.load()
|
||||
except APIError:
|
||||
raise
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise APIError(413, "payload_too_large", "Image dimensions too large") from None
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||
raise APIError(415, "unsupported_media_type", "Invalid image") from None
|
||||
finally:
|
||||
fileobj.seek(0)
|
||||
|
||||
|
||||
def _admit_upload(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest):
|
||||
from .jobs import job_payload
|
||||
stage_key = f"staging/{attachment_id}/original"
|
||||
key = f"attachments/{attachment_id}/original"
|
||||
backup = f"recovery/{attachment_id}/original"
|
||||
job_id = str(uuid.uuid4())
|
||||
outbox_id = str(uuid.uuid4())
|
||||
client = s3_client()
|
||||
try:
|
||||
spool.seek(0)
|
||||
client.upload_fileobj(spool, BUCKET, stage_key,
|
||||
ExtraArgs={"ContentType": mime},
|
||||
Config=TransferConfig(use_threads=False, multipart_threshold=16*1024*1024,
|
||||
io_chunksize=S3_BUFFER_BYTES))
|
||||
except (BotoCoreError, ClientError, EndpointConnectionError, OSError) as exc:
|
||||
raise s3_error(exc) from exc
|
||||
|
||||
# Atomic DB metadata + intent. If DB commit is ambiguous, do not delete the
|
||||
# staged object: an accepted task may already exist; operator cleanup handles it.
|
||||
with db() as conn:
|
||||
with conn.transaction():
|
||||
require_member(conn, chat_id, user_id)
|
||||
# Small educational admission bound; pre-existing queued tasks remain safe.
|
||||
pending = conn.execute("SELECT count(*) AS c FROM processing_jobs WHERE status IN ('queued','processing')").fetchone()["c"]
|
||||
if pending >= int(os.getenv("JOB_BACKLOG_LIMIT", "1000")):
|
||||
raise APIError(429, "rate_limited", "Processing queue is full")
|
||||
row = conn.execute("""
|
||||
INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename,
|
||||
content_type,size_bytes,sha256,status,staging_key,pipeline_version)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,1) RETURNING *
|
||||
""", (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest, stage_key)).fetchone()
|
||||
conn.execute("""
|
||||
INSERT INTO processing_jobs(id,attachment_id,pipeline_version,status,attempt)
|
||||
VALUES (%s,%s,1,'queued',1)
|
||||
""", (job_id, attachment_id))
|
||||
payload = job_payload(job_id, attachment_id, 1, 1)
|
||||
conn.execute("""
|
||||
INSERT INTO job_outbox(id,job_id,payload) VALUES (%s,%s,%s::jsonb)
|
||||
""", (outbox_id, job_id, __import__('json').dumps(payload)))
|
||||
ot.event("upload.accepted", job_id=job_id)
|
||||
return format_attachment(row) # snapshot = queued, even if worker is very fast
|
||||
|
||||
|
||||
@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)):
|
||||
from .api import actor, require_chat
|
||||
who = actor(req)["id"]
|
||||
chat_id = parse_uuid(chat_id)
|
||||
validate_filename(filename)
|
||||
mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if mime not in ("image/png", "image/jpeg"):
|
||||
raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
|
||||
with db() as conn:
|
||||
require_chat(conn, chat_id, who)
|
||||
announced = req.headers.get("content-length")
|
||||
if announced:
|
||||
try:
|
||||
if int(announced) > MAX_BYTES:
|
||||
raise APIError(413, "payload_too_large", "Image too large")
|
||||
if int(announced) < 0:
|
||||
raise invalid("Invalid Content-Length")
|
||||
except ValueError:
|
||||
raise invalid("Invalid Content-Length") from None
|
||||
size = 0
|
||||
digest = hashlib.sha256()
|
||||
with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool:
|
||||
async for chunk in req.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise APIError(413, "payload_too_large", "Image too large")
|
||||
digest.update(chunk)
|
||||
spool.write(chunk)
|
||||
if not size:
|
||||
raise APIError(415, "unsupported_media_type", "Empty image")
|
||||
await run_in_threadpool(validate_image, spool, mime)
|
||||
return await run_in_threadpool(_admit_upload, spool, attachment_id=str(uuid.uuid4()),
|
||||
chat_id=chat_id, user_id=who, filename=filename,
|
||||
mime=mime, size=size, digest=digest.hexdigest())
|
||||
|
||||
|
||||
@router.get("/api/v1/attachments/{attachment_id}")
|
||||
def get_attachment(req: Request, attachment_id: str):
|
||||
who = getattr(req.state, "actor", None) or current_user(req)
|
||||
with db() as conn:
|
||||
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
|
||||
variants = conn.execute("""
|
||||
SELECT name,content_type,size_bytes,width,height,sha256
|
||||
FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s
|
||||
ORDER BY CASE name WHEN 'thumbnail' THEN 0 WHEN 'crop' THEN 1 ELSE 2 END
|
||||
""", (row["id"], row["pipeline_version"])).fetchall() if row["status"] == "ready" else []
|
||||
return format_attachment(row, variants)
|
||||
|
||||
|
||||
@router.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def download_attachment(req: Request, attachment_id: str, variant: str = "original"):
|
||||
who = getattr(req.state, "actor", None) or current_user(req)
|
||||
with db() as conn:
|
||||
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
|
||||
if row["status"] != "ready":
|
||||
raise APIError(409, "conflict", "Attachment is not ready")
|
||||
if variant == "original":
|
||||
key, mime, size = row["object_key"], row["content_type"], row["size_bytes"]
|
||||
else:
|
||||
v = conn.execute("""
|
||||
SELECT * FROM attachment_variants
|
||||
WHERE attachment_id=%s AND pipeline_version=%s AND name=%s
|
||||
""", (row["id"], row["pipeline_version"], variant)).fetchone()
|
||||
if not v:
|
||||
raise APIError(404, "not_found", "Object not found")
|
||||
key, mime, size = v["object_key"], v["content_type"], v["size_bytes"]
|
||||
client = s3_client()
|
||||
try:
|
||||
response = client.get_object(Bucket=BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"):
|
||||
raise APIError(503, "unavailable", "Stored object missing") from exc
|
||||
raise s3_error(exc) from exc
|
||||
except BotoCoreError as exc:
|
||||
raise s3_error(exc) from exc
|
||||
|
||||
def iterator():
|
||||
with closing(response["Body"]) as body:
|
||||
while chunk := body.read(S3_BUFFER_BYTES):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(iterator(), media_type=mime, headers={"Content-Length": str(size)})
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
"""HTTP Basic only on login; bearer sessions shared by all API replicas."""
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError, VerificationError
|
||||
from fastapi import Request
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .core import APIError, SESSION_TTL, iso
|
||||
from .storage import cache, db
|
||||
|
||||
hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
PASSWORD_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
|
||||
|
||||
def public_user(row: dict):
|
||||
return {"id": str(row["id"]), "username": row["username"], "display_name": bytes(row["display_name"]).decode("utf-8"), "created_at": iso(row["created_at"])}
|
||||
|
||||
|
||||
def unauthorized() -> APIError:
|
||||
return APIError(401, "unauthorized", "Unauthorized")
|
||||
|
||||
|
||||
def challenge(req: Request) -> str:
|
||||
return 'Basic realm="messenger", charset="UTF-8"' if req.url.path == "/api/v1/auth/sessions" else 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def bearer_token(req: Request) -> str:
|
||||
value = req.headers.get("Authorization", "")
|
||||
if not value.startswith("Bearer ") or len(value) <= 7 or " " in value[7:] or len(value) > 512:
|
||||
raise unauthorized()
|
||||
try:
|
||||
value[7:].encode("ascii")
|
||||
except UnicodeEncodeError:
|
||||
raise unauthorized() from None
|
||||
return value[7:]
|
||||
|
||||
|
||||
def session_key(token: str) -> str:
|
||||
return "session:" + hashlib.sha256(token.encode("ascii")).hexdigest()
|
||||
|
||||
|
||||
def current_user(req: Request) -> dict:
|
||||
token = bearer_token(req)
|
||||
try:
|
||||
raw = cache().get(session_key(token))
|
||||
except (RedisError, UnicodeError) as exc:
|
||||
raise APIError(503, "unavailable", "Session store unavailable") from exc
|
||||
if not raw:
|
||||
raise unauthorized()
|
||||
try:
|
||||
ident = str(uuid.UUID(raw))
|
||||
except (ValueError, TypeError):
|
||||
try:
|
||||
cache().delete(session_key(token))
|
||||
except RedisError as exc:
|
||||
raise APIError(503, "unavailable", "Session store unavailable") from exc
|
||||
raise unauthorized() from None
|
||||
with db() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE id = %s", (ident,)).fetchone()
|
||||
if row is None:
|
||||
raise unauthorized()
|
||||
return public_user(row)
|
||||
|
||||
|
||||
def login(req: Request):
|
||||
auth = req.headers.get("Authorization", "")
|
||||
if not auth.startswith("Basic "):
|
||||
raise unauthorized()
|
||||
try:
|
||||
decoded = base64.b64decode(auth[6:].encode("ascii"), validate=True).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
if not PASSWORD_RE.fullmatch(username):
|
||||
raise ValueError()
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise unauthorized() from None
|
||||
with db() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
try:
|
||||
hasher.verify(row["password_hash"], password)
|
||||
except (VerifyMismatchError, VerificationError):
|
||||
raise unauthorized() from None
|
||||
token = secrets.token_urlsafe(32)
|
||||
try:
|
||||
cache().set(session_key(token), str(row["id"]), ex=SESSION_TTL)
|
||||
except RedisError as exc:
|
||||
raise APIError(503, "unavailable", "Session store unavailable") from exc
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": iso(datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)), "user": public_user(row)}
|
||||
|
||||
|
||||
def logout(req: Request):
|
||||
current_user(req)
|
||||
try:
|
||||
cache().delete(session_key(bearer_token(req)))
|
||||
except RedisError as exc:
|
||||
raise APIError(503, "unavailable", "Session store unavailable") from exc
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
"""Configuration, HTTP errors and shared representation helpers."""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
INSTANCE_ID = os.getenv("API_INSTANCE_ID", uuid.uuid4().hex[:12])
|
||||
SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
IDEMPOTENCY_TTL = max(86400, int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")))
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
def iso(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def invalid(message: str = "Invalid request") -> APIError:
|
||||
return APIError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def parse_uuid(value: str) -> str:
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise invalid("Invalid UUID") from None
|
||||
|
||||
|
||||
def parse_limit(raw: str | None) -> int:
|
||||
if raw is None:
|
||||
return 50
|
||||
try:
|
||||
if not raw.isascii() or not raw.isdecimal():
|
||||
raise ValueError()
|
||||
value = int(raw)
|
||||
except (ValueError, AttributeError):
|
||||
raise invalid("Invalid limit") from None
|
||||
if not 1 <= value <= 100:
|
||||
raise invalid("Invalid limit")
|
||||
return value
|
||||
|
||||
|
||||
def make_cursor(kind: str, scope: str, created: datetime, ident: str) -> str:
|
||||
b = json.dumps({"kind": kind, "scope": scope, "created_at": iso(created), "id": ident}, separators=(",", ":")).encode()
|
||||
return base64.urlsafe_b64encode(b).decode().rstrip("=")
|
||||
|
||||
|
||||
def read_cursor(raw: str | None, kind: str, scope: str):
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
if not raw or len(raw) > 2048 or any(ch not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" for ch in raw):
|
||||
raise ValueError()
|
||||
data = json.loads(base64.b64decode(raw + "=" * (-len(raw) % 4), altchars=b"-_", validate=True))
|
||||
if not isinstance(data, dict) or data.get("kind") != kind or data.get("scope") != scope:
|
||||
raise ValueError()
|
||||
created = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
|
||||
if created.tzinfo is None:
|
||||
raise ValueError()
|
||||
ident = str(uuid.UUID(data["id"]))
|
||||
return created, ident
|
||||
except (ValueError, TypeError, KeyError, UnicodeError, json.JSONDecodeError):
|
||||
raise invalid("Invalid cursor") from None
|
||||
|
||||
|
||||
def request_id(req: Request) -> str:
|
||||
return getattr(req.state, "request_id", uuid.uuid4().hex)
|
||||
|
||||
|
||||
def error_body(req: Request, code: str, message: str):
|
||||
return {"error": {"code": code, "message": message, "request_id": request_id(req)}}
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Sandboxed-per-task image work. Process is killed after JOB_TIMEOUT_SECONDS.
|
||||
|
||||
Outputs only JSON variant metadata. Original bytes are copied from durable staging,
|
||||
never transformed. Pillow reopens/exif-transposes pixels for derivatives, stripping
|
||||
metadata on re-encode. No API tokens or arbitrary user paths enter this command.
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
|
||||
from .attachments import BUCKET, MAX_BYTES, MAX_PIXELS, s3_client
|
||||
from . import telemetry as ot
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
|
||||
|
||||
def rgb_image(src):
|
||||
orient = ImageOps.exif_transpose(src)
|
||||
orient.load()
|
||||
if orient.width * orient.height > MAX_PIXELS:
|
||||
raise ValueError("too many decoded pixels")
|
||||
# Compose transparency on white, strip EXIF/ICC and do not enlarge.
|
||||
rgba = orient.convert("RGBA")
|
||||
white = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
|
||||
white.alpha_composite(rgba)
|
||||
return white.convert("RGB")
|
||||
|
||||
|
||||
def transform(raw):
|
||||
with Image.open(io.BytesIO(raw)) as image:
|
||||
if image.format not in ("PNG", "JPEG"):
|
||||
raise ValueError("invalid encoding")
|
||||
rgb = rgb_image(image)
|
||||
t = rgb.copy()
|
||||
t.thumbnail((256, 256), resample=Image.Resampling.LANCZOS)
|
||||
# Center crop fit scales up small images when needed.
|
||||
crop = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
|
||||
optim = rgb.copy()
|
||||
optim.thumbnail((1280, 1280), resample=Image.Resampling.LANCZOS)
|
||||
return [
|
||||
("thumbnail", t, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}),
|
||||
("crop", crop, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}),
|
||||
("optimized", optim, "WEBP", "image/webp", {"quality": 78, "method": 4}),
|
||||
]
|
||||
|
||||
|
||||
def run(attachment_id, pipeline_version):
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
with ot.span("db.load_staging_metadata", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5) as conn:
|
||||
row = conn.execute("""
|
||||
SELECT * FROM attachments WHERE id=%s AND pipeline_version=%s
|
||||
""", (attachment_id, pipeline_version)).fetchone()
|
||||
if not row or not row["staging_key"]:
|
||||
raise ValueError("staging metadata not found")
|
||||
|
||||
client = s3_client()
|
||||
body = client.get_object(Bucket=BUCKET, Key=row["staging_key"])["Body"]
|
||||
try:
|
||||
raw = body.read(MAX_BYTES + 1)
|
||||
finally:
|
||||
body.close()
|
||||
if not raw or len(raw) > MAX_BYTES or len(raw) != row["size_bytes"]:
|
||||
raise ValueError("staging size mismatch")
|
||||
if hashlib.sha256(raw).hexdigest() != row["sha256"]:
|
||||
raise ValueError("staging checksum mismatch")
|
||||
|
||||
# Recompute every result if a prior run was interrupted. Keys are deterministic
|
||||
# for (attachment_id,pipeline_version,variant), so duplicates don't multiply objects.
|
||||
output = []
|
||||
for name, image, fmt, mime, options in transform(raw):
|
||||
buf = io.BytesIO()
|
||||
image.save(buf, format=fmt, **options)
|
||||
encoded = buf.getvalue()
|
||||
key = f"variants/{attachment_id}/v{pipeline_version}/{name}"
|
||||
client.put_object(Bucket=BUCKET, Key=key, Body=encoded, ContentType=mime)
|
||||
output.append({"name": name, "content_type": mime,
|
||||
"size_bytes": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(),
|
||||
"width": image.width, "height": image.height, "object_key": key})
|
||||
|
||||
# Only after derivatives exist, publish unchanged primary and recovery copy.
|
||||
client.copy_object(Bucket=BUCKET, Key=row["object_key"],
|
||||
CopySource={"Bucket": BUCKET, "Key": row["staging_key"]},
|
||||
MetadataDirective="REPLACE", ContentType=row["content_type"])
|
||||
client.copy_object(Bucket=BUCKET, Key=row["backup_key"],
|
||||
CopySource={"Bucket": BUCKET, "Key": row["object_key"]},
|
||||
MetadataDirective="REPLACE", ContentType=row["content_type"])
|
||||
return output
|
||||
|
||||
|
||||
def main():
|
||||
import uuid
|
||||
if len(sys.argv) != 3:
|
||||
raise SystemExit(2)
|
||||
try:
|
||||
ident = str(uuid.UUID(sys.argv[1]))
|
||||
version = int(sys.argv[2])
|
||||
if version < 1:
|
||||
raise ValueError("invalid version")
|
||||
ot.init("worker", child=True)
|
||||
with ot.span("image.transform", context=ot.extract(json.loads(os.getenv("LAB6_PARENT_CONTEXT", "{}")))):
|
||||
out = run(ident, version)
|
||||
ot.event("image.transformed")
|
||||
print(json.dumps({"variants": out}, ensure_ascii=False))
|
||||
except (ValueError, UnidentifiedImageError, Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
|
||||
print("Permanent image processing failure", file=sys.stderr)
|
||||
raise SystemExit(20) from None
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
print("Transient object storage failure", file=sys.stderr)
|
||||
raise SystemExit(30) from None
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
finally:
|
||||
ot.flush()
|
||||
+470
@@ -0,0 +1,470 @@
|
||||
"""Durable RabbitMQ delivery + PostgreSQL transactional outbox and worker leases.
|
||||
|
||||
There is no exactly-once transport guarantee. We guarantee eventual recovery of
|
||||
committed intent and idempotent DB results under at-least-once redelivery.
|
||||
Commands: `python -m app.jobs dispatch`, `worker`, `replay --id UUID`, `status`.
|
||||
"""
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
from . import telemetry as ot
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from pika.exceptions import AMQPError
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
log = logging.getLogger("lab5.jobs")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
QUEUE = "attachment-processing.v1"
|
||||
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
|
||||
LEASE_SECONDS = JOB_TIMEOUT + 45
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connect_db():
|
||||
# No credentials, SQL text, or parameters in spans.
|
||||
with ot.dependency_span('db.job.transaction', kind=SpanKind.CLIENT, attrs={'db.system.name':'postgresql'}):
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5, options='-c statement_timeout=5000') as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
def job_payload(job_id, attachment_id, version, attempt):
|
||||
return {
|
||||
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
|
||||
"attachment_id": str(attachment_id), "pipeline_version": version,
|
||||
"attempt": attempt, "created_at": datetime.now(timezone.utc).isoformat(),
|
||||
**ot.inject(),
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection():
|
||||
params = pika.ConnectionParameters(
|
||||
host=os.getenv("BROKER_HOST", "broker"), port=5672,
|
||||
credentials=pika.PlainCredentials(os.environ["BROKER_USER"], os.environ["BROKER_PASSWORD"]),
|
||||
heartbeat=max(120, JOB_TIMEOUT + 60), blocked_connection_timeout=120,
|
||||
socket_timeout=5, connection_attempts=3, retry_delay=1)
|
||||
mq = pika.BlockingConnection(params)
|
||||
try:
|
||||
yield mq
|
||||
finally:
|
||||
try:
|
||||
if mq.is_open:
|
||||
mq.close()
|
||||
except AMQPError:
|
||||
pass
|
||||
|
||||
|
||||
def setup_channel(conn):
|
||||
ch = conn.channel()
|
||||
ch.queue_declare(queue=QUEUE, durable=True)
|
||||
return ch
|
||||
|
||||
|
||||
def queue_outbox(db, job_id, attachment_id, version, attempt, available_at=None):
|
||||
payload = job_payload(job_id, attachment_id, version, attempt)
|
||||
if not payload.get('traceparent'):
|
||||
# Lease rescue/replay has no active consumer context. Preserve the first
|
||||
# acceptance context rather than inventing a new disconnected trace.
|
||||
previous = db.execute("SELECT payload FROM job_outbox WHERE job_id=%s ORDER BY created_at,id LIMIT 1", (job_id,)).fetchone()
|
||||
if previous:
|
||||
payload.update({k: previous['payload'][k] for k in ('traceparent','tracestate') if k in previous['payload']})
|
||||
db.execute("""
|
||||
INSERT INTO job_outbox(id,job_id,payload,available_at)
|
||||
VALUES (%s,%s,%s::jsonb,COALESCE(%s,now()))
|
||||
""", (str(uuid.uuid4()), job_id, json.dumps(payload), available_at))
|
||||
|
||||
|
||||
def cleanup_abandoned_staging():
|
||||
"""Best effort cleanup of retained stages; never delete current processing work."""
|
||||
from .attachments import BUCKET, s3_client
|
||||
client = s3_client()
|
||||
retention = max(3600, int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
|
||||
with connect_db() as db:
|
||||
rows = db.execute("""
|
||||
SELECT id,staging_key FROM attachments
|
||||
WHERE staging_key IS NOT NULL AND
|
||||
((status='ready' AND updated_at < now()-interval '300 seconds') OR
|
||||
(status='failed' AND updated_at < now()-(%s*interval '1 second')))
|
||||
LIMIT 50
|
||||
""", (retention,)).fetchall()
|
||||
for row in rows:
|
||||
with connect_db() as db:
|
||||
with db.transaction():
|
||||
# Row-level lock serializes against operator replay.
|
||||
current = db.execute("SELECT status,staging_key,updated_at FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)).fetchone()
|
||||
if not current or current["staging_key"] != row["staging_key"]:
|
||||
continue
|
||||
age = (datetime.now(timezone.utc) - current["updated_at"]).total_seconds()
|
||||
if not (current["status"] == "ready" and age >= 300 or current["status"] == "failed" and age >= retention):
|
||||
continue
|
||||
client.delete_object(Bucket=BUCKET, Key=row["staging_key"])
|
||||
db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s", (row["id"], row["staging_key"]))
|
||||
# Also prune staging uploads whose DB COMMIT failed, after a long retention.
|
||||
from botocore.exceptions import ClientError
|
||||
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET, Prefix="staging/"):
|
||||
for obj in page.get("Contents", []):
|
||||
if (datetime.now(timezone.utc) - obj["LastModified"]).total_seconds() < retention:
|
||||
continue
|
||||
key = obj["Key"]
|
||||
with connect_db() as db:
|
||||
with db.transaction():
|
||||
referenced = db.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
|
||||
if not referenced:
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
|
||||
def rescue_expired_jobs(conn):
|
||||
"""Requeue leased jobs whose worker died (durably, no manual task recreation)."""
|
||||
with conn.transaction():
|
||||
rows = conn.execute("""
|
||||
SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt
|
||||
FROM processing_jobs j
|
||||
WHERE j.status='processing' AND j.leased_until < now()
|
||||
FOR UPDATE SKIP LOCKED LIMIT 20
|
||||
""").fetchall()
|
||||
for row in rows:
|
||||
if row["attempt"] >= MAX_ATTEMPTS:
|
||||
conn.execute("""UPDATE processing_jobs
|
||||
SET status='failed',error_code='retry_exhausted',lease_token=NULL,leased_until=NULL,updated_at=now()
|
||||
WHERE id=%s""", (row["id"],))
|
||||
conn.execute("""UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now()
|
||||
WHERE id=%s""", (row["attachment_id"],))
|
||||
else:
|
||||
attempt = row["attempt"] + 1
|
||||
conn.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,
|
||||
lease_token=NULL,leased_until=NULL,next_run_at=now(),updated_at=now()
|
||||
WHERE id=%s""", (attempt, row["id"]))
|
||||
conn.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (row["attachment_id"],))
|
||||
queue_outbox(conn, row["id"], row["attachment_id"], row["pipeline_version"], attempt)
|
||||
log.warning("reclaimed timed-out job %s attempt %s", row["id"], attempt)
|
||||
|
||||
|
||||
def sample_backlog():
|
||||
# Query durable *queued* jobs, including delayed retries and unpublished outbox.
|
||||
# No broker membership scan; oldest age is wall clock since initial acceptance.
|
||||
while True:
|
||||
try:
|
||||
with psycopg.connect(os.environ['DATABASE_URL'], row_factory=dict_row, connect_timeout=3, options='-c statement_timeout=2000') as db:
|
||||
row = db.execute("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS age FROM processing_jobs WHERE status='queued'").fetchone()
|
||||
ot.backlog_depth, ot.backlog_age = row['depth'], float(row['age'])
|
||||
except psycopg.Error:
|
||||
ot.backlog_depth = ot.backlog_age = None
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
def dispatch_loop():
|
||||
"""Publish outbox rows with publisher confirm, mark only AFTER confirm."""
|
||||
threading.Thread(target=sample_backlog, daemon=True).start()
|
||||
while True:
|
||||
try:
|
||||
with connection() as mq:
|
||||
ch = setup_channel(mq)
|
||||
ch.confirm_delivery()
|
||||
last_clean = time.monotonic()
|
||||
last_queue_sample = 0
|
||||
while True:
|
||||
# Periodic lease recovery works even if original message was consumed.
|
||||
with connect_db() as conn:
|
||||
rescue_expired_jobs(conn)
|
||||
with conn.transaction():
|
||||
rows = conn.execute("""
|
||||
SELECT id,payload FROM job_outbox
|
||||
WHERE published_at IS NULL AND available_at<=now()
|
||||
ORDER BY available_at,created_at
|
||||
FOR UPDATE SKIP LOCKED LIMIT 20
|
||||
""").fetchall()
|
||||
for row in rows:
|
||||
if os.getenv("LAB5_FAULT_BEFORE_PUBLISH") == "once":
|
||||
# Persist one-time fault before hard exit; outbox still
|
||||
# unmarked and dispatcher will publish on restart.
|
||||
with connect_db() as faultdb:
|
||||
inserted = faultdb.execute("""
|
||||
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'before_publish')
|
||||
ON CONFLICT DO NOTHING RETURNING job_id
|
||||
""", (row["payload"]["job_id"],)).fetchone()
|
||||
if inserted:
|
||||
os._exit(78)
|
||||
payload = row['payload']
|
||||
with ot.span('rabbitmq.publish', kind=SpanKind.PRODUCER, context=ot.extract(payload),
|
||||
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE,'job.attempt':payload['attempt']}) as producer:
|
||||
# Propagate actual producer span, while DB retains the original intent.
|
||||
raw = json.dumps({**payload, **ot.inject()}, separators=(",", ":"))
|
||||
ch.basic_publish(exchange="", routing_key=QUEUE, body=raw.encode("utf-8"),
|
||||
mandatory=True, properties=pika.BasicProperties(content_type="application/json", delivery_mode=2))
|
||||
conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=%s", (row["id"],))
|
||||
ot.event('job.published', job_id=payload['job_id'], attempt=payload['attempt'])
|
||||
if time.monotonic() - last_queue_sample > 3:
|
||||
q = ch.queue_declare(queue=QUEUE, passive=True).method
|
||||
ot.broker_ready, ot.broker_consumers = q.message_count, q.consumer_count
|
||||
last_queue_sample = time.monotonic()
|
||||
if time.monotonic() - last_clean > 60:
|
||||
try:
|
||||
cleanup_abandoned_staging()
|
||||
except Exception:
|
||||
log.warning("staging janitor failed; retry later")
|
||||
last_clean = time.monotonic()
|
||||
if not rows:
|
||||
mq.sleep(1)
|
||||
except (AMQPError, OSError, psycopg.Error) as exc:
|
||||
ot.broker_ready = ot.broker_consumers = None
|
||||
log.warning("dispatcher reconnect after %s", type(exc).__name__)
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def claim_job(db, message):
|
||||
job_id = message["job_id"]
|
||||
with db.transaction():
|
||||
job = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job_id,)).fetchone()
|
||||
if job is None or job["status"] in ("ready", "failed") or job["attempt"] != message["attempt"] or job["pipeline_version"] != message["pipeline_version"]:
|
||||
return "ignore", None
|
||||
if job["status"] == "processing":
|
||||
return "busy", None
|
||||
if job["next_run_at"] > datetime.now(timezone.utc):
|
||||
return "busy", None
|
||||
lease = str(uuid.uuid4())
|
||||
db.execute("""UPDATE processing_jobs SET status='processing',lease_token=%s,
|
||||
leased_until=now()+(%s*interval '1 second'),updated_at=now()
|
||||
WHERE id=%s""", (lease, LEASE_SECONDS, job_id))
|
||||
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (job["attachment_id"],))
|
||||
return "claimed", {"job": job, "lease": lease}
|
||||
|
||||
|
||||
def publish_result(db, *, job, lease, variants):
|
||||
"""Final result is committed atomically; stale leases/versions cannot publish."""
|
||||
with db.transaction():
|
||||
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
|
||||
if current["status"] != "processing" or str(current["lease_token"]) != lease or current["pipeline_version"] != job["pipeline_version"]:
|
||||
log.warning("stale worker result ignored %s", job["id"])
|
||||
return False
|
||||
for v in variants:
|
||||
db.execute("""
|
||||
INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
|
||||
ON CONFLICT(attachment_id,pipeline_version,name) DO UPDATE SET
|
||||
object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type,
|
||||
size_bytes=EXCLUDED.size_bytes,sha256=EXCLUDED.sha256,
|
||||
width=EXCLUDED.width,height=EXCLUDED.height
|
||||
""", (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"],
|
||||
v["content_type"], v["size_bytes"], v["sha256"], v["width"], v["height"]))
|
||||
db.execute("""UPDATE attachments SET status='ready',error_code=NULL,updated_at=now()
|
||||
WHERE id=%s AND pipeline_version=%s""", (job["attachment_id"], job["pipeline_version"]))
|
||||
db.execute("""UPDATE processing_jobs SET status='ready',lease_token=NULL,leased_until=NULL,
|
||||
error_code=NULL,updated_at=now() WHERE id=%s""", (job["id"],))
|
||||
return True
|
||||
|
||||
|
||||
def fail_job(db, job, lease, *, permanent=False, code="processing_failed"):
|
||||
with db.transaction():
|
||||
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
|
||||
if current["status"] != "processing" or str(current["lease_token"]) != lease:
|
||||
return "stale"
|
||||
if permanent or job["attempt"] >= MAX_ATTEMPTS:
|
||||
code = code if permanent else "retry_exhausted"
|
||||
db.execute("""UPDATE processing_jobs SET status='failed',error_code=%s,lease_token=NULL,
|
||||
leased_until=NULL,updated_at=now() WHERE id=%s""", (code, job["id"]))
|
||||
db.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s", (code, job["attachment_id"]))
|
||||
return "failed"
|
||||
else:
|
||||
attempt = job["attempt"] + 1
|
||||
delay = min(60.0, 2.0 * (2 ** (attempt - 2))) + random.uniform(0, 1.0)
|
||||
run_at = datetime.now(timezone.utc) + timedelta(seconds=delay)
|
||||
db.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,next_run_at=%s,
|
||||
lease_token=NULL,leased_until=NULL,updated_at=now() WHERE id=%s""", (attempt, run_at, job["id"]))
|
||||
db.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (job["attachment_id"],))
|
||||
queue_outbox(db, job["id"], job["attachment_id"], job["pipeline_version"], attempt, run_at)
|
||||
log.warning("job %s scheduled retry %s after %.1fs", job["id"], attempt, delay)
|
||||
return "retry"
|
||||
|
||||
|
||||
def cleanup_staging(job):
|
||||
from .attachments import BUCKET, s3_client
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
try:
|
||||
with connect_db() as conn:
|
||||
row = conn.execute("SELECT staging_key,status FROM attachments WHERE id=%s", (job["attachment_id"],)).fetchone()
|
||||
if row and row["status"] == "ready" and row["staging_key"]:
|
||||
s3_client().delete_object(Bucket=BUCKET, Key=row["staging_key"])
|
||||
with connect_db() as conn:
|
||||
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND status='ready' AND staging_key=%s",
|
||||
(job["attachment_id"], row["staging_key"]))
|
||||
except (BotoCoreError, ClientError, psycopg.Error, OSError):
|
||||
log.warning("staging cleanup deferred for %s", job["id"])
|
||||
|
||||
|
||||
def process_message(raw):
|
||||
try:
|
||||
carrier = json.loads(raw)
|
||||
if not isinstance(carrier, dict): carrier = {}
|
||||
except (ValueError, TypeError):
|
||||
carrier = {}
|
||||
with ot.span('rabbitmq.consume', kind=SpanKind.CONSUMER, context=ot.extract(carrier),
|
||||
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE}) as consumer:
|
||||
return process_message_inner(raw, consumer)
|
||||
|
||||
|
||||
def process_message_inner(raw, consumer):
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
assert msg["schema_version"] == 1 and msg["type"] == "attachment.process.v1"
|
||||
uuid.UUID(msg["job_id"])
|
||||
uuid.UUID(msg["attachment_id"])
|
||||
assert type(msg["attempt"]) is int and msg["attempt"] >= 1
|
||||
except (ValueError, KeyError, AssertionError, TypeError):
|
||||
log.error("invalid internal broker message discarded")
|
||||
return "ack" # no secrets printed, does not touch DB
|
||||
with connect_db() as db:
|
||||
status, state = claim_job(db, msg)
|
||||
if status == "ignore":
|
||||
return "ack"
|
||||
if status == "busy":
|
||||
return "requeue"
|
||||
job, lease = state["job"], state["lease"]
|
||||
consumer.set_attribute('job.attempt',job['attempt'])
|
||||
consumer.set_attribute('job.id',str(job['id']))
|
||||
ot.event('job.claimed', job_id=str(job['id']), attempt=job['attempt'])
|
||||
start = time.monotonic()
|
||||
ot.worker_busy = 1
|
||||
def done(outcome):
|
||||
attrs = {'job_type':'attachment.process.v1','outcome':outcome}
|
||||
ot.count('lab_jobs',attrs)
|
||||
ot.observe('lab_job_duration',time.monotonic()-start,attrs)
|
||||
if outcome == 'retry':
|
||||
ot.count('lab_job_retries',{'job_type':'attachment.process.v1'})
|
||||
if outcome in ('retry','failed','requeue'):
|
||||
consumer.set_status(Status(StatusCode.ERROR))
|
||||
consumer.set_attribute('job.outcome',outcome)
|
||||
ot.event('job.'+outcome, level=logging.ERROR if outcome in ('failed','retry','requeue') else logging.INFO, job_id=str(job['id']), attempt=job['attempt'])
|
||||
try:
|
||||
if os.getenv('LAB6_FAULT_RETRY_ONCE') == 'once':
|
||||
with connect_db() as db:
|
||||
inserted = db.execute("INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'lab6_retry') ON CONFLICT DO NOTHING RETURNING job_id", (job['id'],)).fetchone()
|
||||
if inserted:
|
||||
raise RuntimeError('injected transient retry')
|
||||
cmd = [sys.executable, "-m", "app.image_worker", str(job["attachment_id"]), str(job["pipeline_version"])]
|
||||
with ot.span('image.subprocess'):
|
||||
env = {**os.environ, 'LAB6_PARENT_CONTEXT': json.dumps(ot.inject())}
|
||||
proc = subprocess.run(cmd, capture_output=True, timeout=JOB_TIMEOUT, check=False, env=env)
|
||||
if proc.returncode != 0:
|
||||
with connect_db() as db:
|
||||
outcome = fail_job(db, job, lease, permanent=(proc.returncode == 20),
|
||||
code="invalid_image" if proc.returncode == 20 else "processing_failed")
|
||||
done(outcome)
|
||||
return "ack" # next attempt is already safely in DB outbox
|
||||
result = json.loads(proc.stdout)
|
||||
variants = result["variants"]
|
||||
if sorted(v["name"] for v in variants) != ["crop", "optimized", "thumbnail"]:
|
||||
raise ValueError("incomplete variants")
|
||||
if os.getenv("LAB5_FAULT_AFTER_S3") == "once":
|
||||
# One-time operator-only injected hard death, after all S3 writes and
|
||||
# before DB result commit/ack; SQL PK prevents endless failure loop.
|
||||
with connect_db() as db:
|
||||
with db.transaction():
|
||||
inserted = db.execute("""
|
||||
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'after_s3')
|
||||
ON CONFLICT DO NOTHING RETURNING job_id
|
||||
""", (job["id"],)).fetchone()
|
||||
if inserted:
|
||||
os._exit(77)
|
||||
with ot.span("db.publish_result", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
|
||||
with connect_db() as db:
|
||||
ok = publish_result(db, job=job, lease=lease, variants=variants)
|
||||
if ok:
|
||||
ot.observe('lab_ready_duration',(datetime.now(timezone.utc)-job['created_at']).total_seconds(), {'job_type':'attachment.process.v1'})
|
||||
cleanup_staging(job)
|
||||
done('ready' if ok else 'stale')
|
||||
return "ack"
|
||||
except subprocess.TimeoutExpired:
|
||||
with connect_db() as db:
|
||||
outcome = fail_job(db, job, lease, code="processing_timeout")
|
||||
done(outcome)
|
||||
return "ack"
|
||||
except Exception:
|
||||
log.exception("job processing failure (internal details not sent to API)")
|
||||
# Leave lease for dispatcher recovery if even DB unavailable.
|
||||
try:
|
||||
with connect_db() as db:
|
||||
outcome = fail_job(db, job, lease)
|
||||
done(outcome)
|
||||
return "ack"
|
||||
except psycopg.Error:
|
||||
done("requeue")
|
||||
return "requeue"
|
||||
finally:
|
||||
ot.worker_busy = 0
|
||||
|
||||
|
||||
def worker_loop():
|
||||
while True:
|
||||
try:
|
||||
with connection() as mq:
|
||||
ch = setup_channel(mq)
|
||||
ch.basic_qos(prefetch_count=max(1, int(os.getenv("WORKER_PREFETCH", "1"))))
|
||||
|
||||
def callback(channel, method, props, body):
|
||||
outcome = process_message(body)
|
||||
if outcome == "ack":
|
||||
channel.basic_ack(method.delivery_tag)
|
||||
else:
|
||||
channel.basic_nack(method.delivery_tag, requeue=True)
|
||||
time.sleep(2) # prevent tight redelivery spin on an active lease
|
||||
|
||||
ch.basic_consume(queue=QUEUE, on_message_callback=callback, auto_ack=False)
|
||||
log.info("worker ready; max attempts=%d timeout=%ds", MAX_ATTEMPTS, JOB_TIMEOUT)
|
||||
ch.start_consuming()
|
||||
except (AMQPError, OSError, psycopg.Error) as exc:
|
||||
log.warning("worker reconnect after %s", type(exc).__name__)
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def replay_failed(ident):
|
||||
with connect_db() as db:
|
||||
with db.transaction():
|
||||
row = db.execute("SELECT * FROM processing_jobs WHERE attachment_id=%s FOR UPDATE", (ident,)).fetchone()
|
||||
if not row or row["status"] != "failed":
|
||||
raise RuntimeError("Only failed jobs may be explicitly replayed")
|
||||
attachment = db.execute("SELECT staging_key FROM attachments WHERE id=%s", (ident,)).fetchone()
|
||||
if not attachment or not attachment["staging_key"]:
|
||||
raise RuntimeError("Retained staging is required for replay")
|
||||
db.execute("""UPDATE processing_jobs SET status='queued',attempt=1,next_run_at=now(),
|
||||
lease_token=NULL,leased_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s""", (row["id"],))
|
||||
db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s", (ident,))
|
||||
queue_outbox(db, row["id"], ident, row["pipeline_version"], 1)
|
||||
db.execute("INSERT INTO job_replay_audit(id,job_id) VALUES (%s,%s)", (str(uuid.uuid4()), row["id"]))
|
||||
print("Replayed failed job", ident)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("mode", choices=["dispatch", "worker", "replay", "status"])
|
||||
parser.add_argument("--id", help="UUID for explicit operator replay/status")
|
||||
args = parser.parse_args()
|
||||
if args.mode == "dispatch":
|
||||
ot.init("dispatcher")
|
||||
dispatch_loop()
|
||||
elif args.mode == "worker":
|
||||
ot.init("worker")
|
||||
worker_loop()
|
||||
elif args.mode in ("replay", "status"):
|
||||
if not args.id:
|
||||
parser.error("--id is required")
|
||||
ident = str(uuid.UUID(args.id))
|
||||
if args.mode == "replay":
|
||||
replay_failed(ident)
|
||||
else:
|
||||
with connect_db() as db:
|
||||
print(json.dumps(db.execute("SELECT status,attempt,error_code FROM processing_jobs WHERE attachment_id=%s", (ident,)).fetchone(), default=str))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
"""FastAPI process: one stateless replica. No auto-migration on startup."""
|
||||
import logging
|
||||
import uuid
|
||||
import time
|
||||
from starlette.routing import Match
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from . import telemetry as ot
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import psycopg
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .api import router
|
||||
from .attachments import router as attachment_router
|
||||
from .auth import challenge, current_user
|
||||
from .core import APIError, INSTANCE_ID, error_body, invalid, request_id
|
||||
from .storage import start, stop
|
||||
|
||||
logging.basicConfig(level="INFO")
|
||||
logger = logging.getLogger("lab5")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
ot.init("api")
|
||||
start()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
stop()
|
||||
ot.flush()
|
||||
|
||||
app = FastAPI(title="MTUCI Messenger Lab 6", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan)
|
||||
app.include_router(router)
|
||||
app.include_router(attachment_router)
|
||||
|
||||
PROTECTED = ("/api/v1/chats", "/api/v1/attachments", "/api/v1/users/me", "/api/v1/auth/sessions/current")
|
||||
JSON_POST = ("/api/v1/users", "/api/v1/chats")
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_context(req: Request, call_next):
|
||||
# Match the registered template even when early authentication fails.
|
||||
route = next((r.path for r in app.routes if r.matches(req.scope)[0] == Match.FULL), 'unmatched')
|
||||
method = req.method if req.method in ('GET','POST','DELETE','PUT','PATCH','HEAD','OPTIONS') else 'OTHER'
|
||||
attrs = {'method': method, 'route': route}
|
||||
start_at = time.monotonic()
|
||||
if route in ("/health/live", "/health/ready"):
|
||||
response = await handle_request(req, call_next)
|
||||
attrs["status_class"] = str(response.status_code//100)+"xx"
|
||||
ot.count("lab_http_requests", attrs)
|
||||
ot.observe("lab_http_duration", time.monotonic()-start_at, attrs)
|
||||
return response
|
||||
with ot.span(method+' '+route, kind=SpanKind.SERVER, attrs={'http.request.method':method,'http.route':route}) as span:
|
||||
response = await handle_request(req, call_next)
|
||||
attrs['status_class'] = str(response.status_code//100)+'xx'
|
||||
span.set_attribute('http.response.status_code', response.status_code)
|
||||
if response.status_code >= 500:
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
ot.count('lab_http_requests', attrs)
|
||||
ot.observe('lab_http_duration', time.monotonic()-start_at, attrs)
|
||||
ot.event('http.completed', level=logging.ERROR if response.status_code >= 500 else logging.INFO,
|
||||
request_id=request_id(req), route=route, method=method, status_code=response.status_code)
|
||||
if ot.trace_id():
|
||||
response.headers['X-Trace-Id'] = ot.trace_id()
|
||||
return response
|
||||
|
||||
|
||||
async def handle_request(req: Request, call_next):
|
||||
req.state.request_id = uuid.uuid4().hex
|
||||
try:
|
||||
path = req.url.path
|
||||
# Authenticate before parsing the request body or resource-specific fields.
|
||||
if any(path == p or path.startswith(p + "/") for p in PROTECTED):
|
||||
req.state.actor = await run_in_threadpool(current_user, req)
|
||||
if req.method == "POST" and (path in JSON_POST or (path.startswith("/api/v1/chats/") and path.endswith("/messages"))):
|
||||
if req.headers.get("content-type", "").split(";", 1)[0].strip().lower() != "application/json":
|
||||
raise APIError(415, "unsupported_media_type", "Content-Type must be application/json")
|
||||
if req.method == "POST" and path == "/api/v1/auth/sessions" and await req.body():
|
||||
raise invalid("Login body is not allowed")
|
||||
response = await call_next(req)
|
||||
except APIError as exc:
|
||||
response = api_error(req, exc)
|
||||
except (RedisError, psycopg.OperationalError, psycopg.InterfaceError):
|
||||
response = api_error(req, APIError(503, "unavailable", "Dependency unavailable"))
|
||||
except Exception:
|
||||
logger.exception("unexpected request failure id=%s", request_id(req))
|
||||
response = api_error(req, APIError(500, "internal_error", "Internal server error"))
|
||||
response.headers["X-Request-Id"] = request_id(req)
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
def api_error(req: Request, exc: APIError) -> JSONResponse:
|
||||
headers = {"WWW-Authenticate": challenge(req)} if exc.status == 401 else {}
|
||||
return JSONResponse(status_code=exc.status, content=error_body(req, exc.code, exc.message), headers=headers)
|
||||
|
||||
|
||||
@app.exception_handler(APIError)
|
||||
async def api_error_handler(req: Request, exc: APIError):
|
||||
return api_error(req, exc)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(req: Request, exc: RequestValidationError):
|
||||
return api_error(req, invalid())
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error_handler(req: Request, exc: Exception):
|
||||
logger.exception("handler failure id=%s", request_id(req))
|
||||
if isinstance(exc, (psycopg.OperationalError, psycopg.InterfaceError, RedisError)):
|
||||
return api_error(req, APIError(503, "unavailable", "Dependency unavailable"))
|
||||
return api_error(req, APIError(500, "internal_error", "Internal server error"))
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Run once before API instances; transactional and advisory-lock coordinated migrations."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
|
||||
MIGRATIONS = Path(__file__).resolve().parents[1] / "db" / "migrations"
|
||||
|
||||
def main():
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], autocommit=False) as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(62132103)")
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())")
|
||||
for path in sorted(MIGRATIONS.glob("[0-9]*.sql")):
|
||||
applied = conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone()
|
||||
if applied:
|
||||
continue
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (path.name,))
|
||||
print("Applied", path.name, flush=True)
|
||||
print("Migrations up to date", flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,77 @@
|
||||
"""HTTP request models; all strings use Unicode code point limits."""
|
||||
import re
|
||||
import uuid
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
class CreateUser(StrictModel):
|
||||
username: str
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def username_valid(cls, value: str) -> str:
|
||||
if not USERNAME_RE.fullmatch(value):
|
||||
raise ValueError("Invalid username")
|
||||
return value
|
||||
|
||||
@field_validator("display_name", "password")
|
||||
@classmethod
|
||||
def non_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("Blank value")
|
||||
value.encode("utf-8", errors="strict")
|
||||
return value
|
||||
|
||||
class CreateChat(StrictModel):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def non_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("Blank title")
|
||||
value.encode("utf-8", errors="strict")
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def uuid_members(cls, value: list[str]) -> list[str]:
|
||||
for member in value:
|
||||
if not isinstance(member, str) or str(uuid.UUID(member)) != member:
|
||||
raise ValueError("Invalid member UUID")
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("Duplicate member")
|
||||
return value
|
||||
|
||||
class CreateMessage(StrictModel):
|
||||
text: str = Field(min_length=0, max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("text")
|
||||
@classmethod
|
||||
def non_blank(cls, value: str) -> str:
|
||||
value.encode("utf-8", errors="strict")
|
||||
return value
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def valid_attachments(cls, value: list[str]) -> list[str]:
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("Duplicate attachment IDs")
|
||||
for item in value:
|
||||
if not isinstance(item, str) or str(uuid.UUID(item)) != item:
|
||||
raise ValueError("Invalid attachment UUID")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def nonempty_message(self):
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError("Empty message")
|
||||
return self
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Offline image conversion tests inside the application image; no S3 or DB."""
|
||||
import io
|
||||
from PIL import Image
|
||||
from .image_worker import transform
|
||||
|
||||
|
||||
def main():
|
||||
source = Image.new('RGB',(320,200), (255,0,0))
|
||||
buf=io.BytesIO();source.save(buf,'PNG')
|
||||
result=transform(buf.getvalue())
|
||||
items={name:(img,fmt,mime,opts) for name,img,fmt,mime,opts in result}
|
||||
assert set(items)=={'thumbnail','crop','optimized'}
|
||||
assert items['thumbnail'][0].size==(256,160)
|
||||
assert items['crop'][0].size==(128,128)
|
||||
assert items['optimized'][0].size==(320,200)
|
||||
for name,(im,fmt,mime,opts) in items.items():
|
||||
data=io.BytesIO();im.save(data,format=fmt,**opts)
|
||||
with Image.open(io.BytesIO(data.getvalue())) as decoded:
|
||||
assert decoded.size==im.size
|
||||
assert decoded.format==fmt
|
||||
assert 'exif' not in decoded.info
|
||||
print('PASS: PNG 320x200 -> JPEG thumb 256x160, crop 128x128, optimized WebP, metadata stripped')
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Lab4 grade-5 track 2: repeatable DB/S3 reconciliation.
|
||||
|
||||
Dry-run is default. Apply never drops metadata when BOTH object copies are gone:
|
||||
without a trustworthy copy or external backup, deletion would destroy evidence.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .attachments import BUCKET, s3_client
|
||||
|
||||
|
||||
def inventory(client):
|
||||
result = {}
|
||||
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET):
|
||||
for obj in page.get("Contents", []):
|
||||
# Only reconciliation-owned prefixes; never mutate arbitrary S3 keys.
|
||||
key = obj["Key"]
|
||||
if key.startswith(("attachments/", "recovery/")):
|
||||
result[key] = obj["LastModified"]
|
||||
return result
|
||||
|
||||
|
||||
def get_db_rows(conn):
|
||||
return conn.execute("SELECT id, object_key, backup_key, status FROM attachments").fetchall()
|
||||
|
||||
|
||||
def current_reference(conn, key):
|
||||
return conn.execute(
|
||||
"SELECT 1 FROM attachments WHERE object_key=%s OR backup_key=%s", (key, key)
|
||||
).fetchone() is not None
|
||||
|
||||
|
||||
def reconcile(*, apply=False, grace_seconds=300, now=None, client=None, connect=None):
|
||||
if grace_seconds < 300 and apply:
|
||||
raise ValueError("Apply requires >=300 s grace period to protect in-flight uploads")
|
||||
started = time.monotonic()
|
||||
now = now or datetime.now(timezone.utc)
|
||||
client = client or s3_client()
|
||||
connect = connect or (lambda: psycopg.connect(__import__('os').environ['DATABASE_URL'], row_factory=dict_row))
|
||||
objects = inventory(client)
|
||||
stats = {
|
||||
"mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds,
|
||||
"s3_keys_scanned": len(objects), "db_rows_scanned": 0,
|
||||
"orphan_keys": [], "orphan_keys_young": [],
|
||||
"missing_primary_repairable": [], "missing_backup_repairable": [],
|
||||
"missing_both_unrecoverable": [], "actions": [], "errors": [],
|
||||
}
|
||||
with connect() as conn:
|
||||
rows = get_db_rows(conn)
|
||||
stats["db_rows_scanned"] = len(rows)
|
||||
references = {key for row in rows for key in (row["object_key"], row["backup_key"])}
|
||||
for row in rows:
|
||||
if row["status"] != "ready":
|
||||
continue # queued/processing have durable staging, not final variants yet
|
||||
primary, backup = row["object_key"], row["backup_key"]
|
||||
have_primary, have_backup = primary in objects, backup in objects
|
||||
rid = str(row["id"])
|
||||
if have_primary and have_backup:
|
||||
continue
|
||||
if not have_primary and not have_backup:
|
||||
stats["missing_both_unrecoverable"].append(rid)
|
||||
continue
|
||||
dest, source, typ = (primary, backup, "missing_primary_repairable") if not have_primary else (backup, primary, "missing_backup_repairable")
|
||||
stats[typ].append(rid)
|
||||
if apply:
|
||||
# Defensive recheck. Object may have reappeared after inventory.
|
||||
try:
|
||||
client.head_object(Bucket=BUCKET, Key=dest)
|
||||
stats["actions"].append({"action": "already-restored", "key": dest})
|
||||
continue
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code")
|
||||
if code not in ("404", "NotFound", "NoSuchKey"):
|
||||
raise
|
||||
try:
|
||||
client.copy_object(Bucket=BUCKET, Key=dest,
|
||||
CopySource={"Bucket": BUCKET, "Key": source})
|
||||
stats["actions"].append({"action": "restore", "key": dest, "from": source})
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
stats["errors"].append({"action": "restore", "key": dest, "error_type": type(exc).__name__})
|
||||
for key, modified in objects.items():
|
||||
if key in references:
|
||||
continue
|
||||
age = (now - modified).total_seconds()
|
||||
if age < grace_seconds:
|
||||
stats["orphan_keys_young"].append(key)
|
||||
continue
|
||||
stats["orphan_keys"].append(key)
|
||||
if apply:
|
||||
# Query again immediately before destructive action; never delete
|
||||
# an object now referenced by a committed metadata row.
|
||||
if current_reference(conn, key):
|
||||
stats["actions"].append({"action": "skip-now-referenced", "key": key})
|
||||
continue
|
||||
try:
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
stats["actions"].append({"action": "delete-old-orphan", "key": key})
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
stats["errors"].append({"action": "delete-old-orphan", "key": key, "error_type": type(exc).__name__})
|
||||
stats["scan_elapsed_seconds"] = round(time.monotonic() - started, 3)
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Reconcile S3 attachments vs PostgreSQL (dry-run default)")
|
||||
parser.add_argument("--apply", action="store_true", help="Make safe changes; orphans older than 300 seconds only")
|
||||
parser.add_argument("--grace-seconds", type=int, default=300)
|
||||
parser.add_argument("--output", help="Write JSON summary (no credentials)")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = reconcile(apply=args.apply, grace_seconds=args.grace_seconds)
|
||||
except Exception as exc:
|
||||
# No exception string: botocore errors may contain signed credentials/URLs.
|
||||
print(json.dumps({"status":"failed", "error_type":type(exc).__name__}), file=sys.stderr)
|
||||
raise SystemExit(2) from None
|
||||
rendered = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
print(rendered)
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as fh:
|
||||
fh.write(rendered + "\n")
|
||||
if result["errors"] or result["missing_both_unrecoverable"]:
|
||||
raise SystemExit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Operator-only fault injection in a LOCAL TEST stack; never exposed as HTTP."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from .attachments import BUCKET, s3_client
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("action", choices=["delete-primary", "delete-backup", "create-orphan"])
|
||||
p.add_argument("--id", help="Attachment UUID for the delete actions")
|
||||
args = p.parse_args()
|
||||
if os.environ.get("LAB4_ALLOW_FAULT_INJECTION") != "yes":
|
||||
raise SystemExit("Fault injection disabled; use LAB4_ALLOW_FAULT_INJECTION=yes in isolated test")
|
||||
client = s3_client()
|
||||
if args.action == "create-orphan":
|
||||
key = f"attachments/{uuid.uuid4()}/original"
|
||||
client.put_object(Bucket=BUCKET, Key=key, Body=b"synthetic orphan")
|
||||
print(json.dumps({"fault":"orphan-created", "object_key":key}))
|
||||
return
|
||||
if not args.id:
|
||||
raise SystemExit("--id is required")
|
||||
ident = str(uuid.UUID(args.id))
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row) as conn:
|
||||
row = conn.execute("SELECT object_key,backup_key FROM attachments WHERE id=%s", (ident,)).fetchone()
|
||||
if not row:
|
||||
raise SystemExit("Attachment does not exist")
|
||||
key = row["object_key"] if args.action == "delete-primary" else row["backup_key"]
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
print(json.dumps({"fault":args.action, "object_key":key}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Fast isolated tests for reconciliation safety; run in application image."""
|
||||
import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from .reconcile import reconcile
|
||||
from .attachments import BUCKET
|
||||
|
||||
NOW = datetime.now(timezone.utc)
|
||||
|
||||
class FakeConn:
|
||||
def __init__(self, rows): self.rows=rows
|
||||
def __enter__(self): return self
|
||||
def __exit__(self,*a): return False
|
||||
def execute(self, sql, args=None):
|
||||
self.sql=sql
|
||||
self.args=args
|
||||
return self
|
||||
def fetchall(self): return self.rows
|
||||
def fetchone(self):
|
||||
key=self.args[0]
|
||||
return {'id': 1} if any(key in (row['object_key'],row['backup_key']) for row in self.rows) else None
|
||||
|
||||
class FakeS3:
|
||||
def __init__(self,objects): self.objects=objects
|
||||
def get_paginator(self,op): return self
|
||||
def paginate(self,**kwargs):
|
||||
yield {'Contents':[{'Key':k,'LastModified':t} for k,t in self.objects.items()]}
|
||||
def head_object(self,**kw):
|
||||
if kw['Key'] not in self.objects:
|
||||
from botocore.exceptions import ClientError
|
||||
raise ClientError({'Error':{'Code':'404'}},'HeadObject')
|
||||
def copy_object(self,**kw):
|
||||
assert kw['CopySource']['Key'] in self.objects
|
||||
self.objects[kw['Key']]=NOW
|
||||
def delete_object(self,**kw):
|
||||
self.objects.pop(kw['Key'],None)
|
||||
|
||||
|
||||
def test():
|
||||
i=str(uuid.uuid4())
|
||||
primary=f'attachments/{i}/original';backup=f'recovery/{i}/original'
|
||||
young=f'attachments/{uuid.uuid4()}/original'
|
||||
old=f'attachments/{uuid.uuid4()}/original'
|
||||
rows=[{'id':i,'object_key':primary,'backup_key':backup,'status':'ready'}]
|
||||
fake=FakeS3({backup: NOW-timedelta(days=1),young:NOW-timedelta(seconds=10),old:NOW-timedelta(days=1)})
|
||||
kwargs={'now':NOW,'client':fake,'connect':lambda:FakeConn(rows)}
|
||||
first=reconcile(apply=False,**kwargs)
|
||||
assert first['missing_primary_repairable']==[i]
|
||||
assert old in first['orphan_keys'] and young in first['orphan_keys_young']
|
||||
assert primary not in fake.objects and old in fake.objects, 'Dry run mutated objects'
|
||||
second=reconcile(apply=True,**kwargs)
|
||||
assert primary in fake.objects and old not in fake.objects and young in fake.objects
|
||||
assert any(x['action']=='restore' for x in second['actions'])
|
||||
third=reconcile(apply=True,**kwargs)
|
||||
assert not third['actions'], 'Re-run unexpectedly mutated objects'
|
||||
assert rows[0]['object_key']==primary, 'Metadata destroyed'
|
||||
try: reconcile(apply=True, grace_seconds=0, **kwargs)
|
||||
except ValueError: pass
|
||||
else: raise AssertionError('Unsafe zero-grace apply accepted')
|
||||
print('PASS: dry-run, orphan grace, recover primary from backup, apply idempotence')
|
||||
|
||||
if __name__ == '__main__': test()
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Idempotently create the private S3 bucket with bootstrap-only credentials."""
|
||||
import os
|
||||
import time
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
|
||||
|
||||
def main():
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
client = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
aws_access_key_id=os.environ["S3_ADMIN_ACCESS_KEY"],
|
||||
aws_secret_access_key=os.environ["S3_ADMIN_SECRET_KEY"],
|
||||
region_name="us-east-1",
|
||||
config=Config(s3={"addressing_style": "path"},
|
||||
connect_timeout=2, read_timeout=4, retries={"max_attempts": 1}))
|
||||
for attempt in range(70):
|
||||
try:
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code not in ("404", "NoSuchBucket", "NotFound"):
|
||||
raise
|
||||
client.create_bucket(Bucket=bucket)
|
||||
# PRIVATE: no public bucket policy, ACL or anonymous identity.
|
||||
print(f"Private bucket {bucket} ready", flush=True)
|
||||
return
|
||||
except (ClientError, BotoCoreError, OSError) as exc:
|
||||
if attempt == 69:
|
||||
raise RuntimeError("S3 bootstrap failed; inspect S3 container") from exc
|
||||
time.sleep(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
"""One-off creation of private SeaweedFS S3 identities; secrets stay in Docker volume."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
identities = [
|
||||
{"name": "bucket-bootstrap", "credentials": [{"accessKey": os.environ["S3_ADMIN_ACCESS_KEY"],
|
||||
"secretKey": os.environ["S3_ADMIN_SECRET_KEY"]}],
|
||||
"actions": ["Admin", "Read", "Write", "List"]},
|
||||
{"name": "messenger-api", "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY"],
|
||||
"secretKey": os.environ["S3_SECRET_KEY"]}],
|
||||
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]},
|
||||
]
|
||||
if len({entry["credentials"][0]["accessKey"] for entry in identities}) != 2:
|
||||
raise ValueError("Admin and API credentials must differ")
|
||||
path = Path("/s3-config/config.json")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
old_umask = os.umask(0o077)
|
||||
try:
|
||||
tmp.write_text(json.dumps({"identities": identities}, separators=(",", ":")), encoding="utf-8")
|
||||
os.chmod(tmp, 0o644) # readable by the SeaweedFS runtime UID; Docker volume stays private
|
||||
os.replace(tmp, path)
|
||||
finally:
|
||||
os.umask(old_umask)
|
||||
print("S3 identities configured (no secrets printed)", flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Verify bucket is private at S3 boundary (not merely hidden behind HTTP API)."""
|
||||
import json
|
||||
import os
|
||||
import boto3
|
||||
from botocore import UNSIGNED
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from .attachments import BUCKET, s3_client
|
||||
|
||||
|
||||
def main():
|
||||
client = s3_client()
|
||||
client.head_bucket(Bucket=BUCKET) # restricted app identity works
|
||||
unauth = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
region_name="us-east-1", config=Config(signature_version=UNSIGNED,
|
||||
s3={"addressing_style":"path"}))
|
||||
try:
|
||||
unauth.list_objects_v2(Bucket=BUCKET, MaxKeys=1)
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
if status not in (401,403):
|
||||
raise RuntimeError("Unexpected anonymous S3 response") from None
|
||||
else:
|
||||
raise RuntimeError("Private bucket leaked object list to anonymous caller")
|
||||
listing = client.list_objects_v2(Bucket=BUCKET, MaxKeys=1)
|
||||
keys = [entry["Key"] for entry in listing.get("Contents", [])]
|
||||
if keys:
|
||||
try:
|
||||
unauth.get_object(Bucket=BUCKET, Key=keys[0])
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
if status not in (401, 403):
|
||||
raise RuntimeError("Unexpected anonymous object GET response") from None
|
||||
else:
|
||||
raise RuntimeError("Private S3 object downloadable anonymously")
|
||||
print(json.dumps({"private_bucket": True, "anonymous_listing": "denied",
|
||||
"anonymous_object_get": "denied" if keys else "not_tested_empty_bucket"}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Shared external state: psycopg connection pool and Redis session store."""
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
from redis import Redis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .core import APIError
|
||||
from . import telemetry as ot
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
pool: ConnectionPool | None = None
|
||||
redis_client: Redis | None = None
|
||||
|
||||
|
||||
def start() -> None:
|
||||
global pool, redis_client
|
||||
url = os.environ["DATABASE_URL"]
|
||||
size = int(os.getenv("DB_POOL_MAX", "8"))
|
||||
pool = ConnectionPool(conninfo=url, min_size=1, max_size=size, timeout=3, kwargs={"row_factory": dict_row, "connect_timeout": 5, "options": "-c statement_timeout=5000"}, open=False)
|
||||
pool.open(wait=True, timeout=30)
|
||||
redis_client = Redis.from_url(os.environ.get("REDIS_URL", "redis://redis:6379/0"), socket_connect_timeout=2, socket_timeout=2, decode_responses=True)
|
||||
redis_client.ping()
|
||||
|
||||
|
||||
def stop() -> None:
|
||||
global pool, redis_client
|
||||
if pool is not None:
|
||||
pool.close()
|
||||
pool = None
|
||||
if redis_client is not None:
|
||||
redis_client.close()
|
||||
redis_client = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def db():
|
||||
if pool is None:
|
||||
raise APIError(503, "unavailable", "Database unavailable")
|
||||
try:
|
||||
with ot.dependency_span("db.transaction", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
|
||||
with pool.connection() as conn:
|
||||
yield conn
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled, PoolTimeout) as exc:
|
||||
raise APIError(503, "unavailable", "Database unavailable") from exc
|
||||
|
||||
|
||||
def cache() -> Redis:
|
||||
if redis_client is None:
|
||||
raise APIError(503, "unavailable", "Session store unavailable")
|
||||
return redis_client
|
||||
|
||||
|
||||
def ready() -> bool:
|
||||
try:
|
||||
with db() as conn:
|
||||
conn.execute("SELECT 1").fetchone()
|
||||
cache().ping()
|
||||
# S3 is needed for the mandatory synchronous upload/download path.
|
||||
from .attachments import s3_client, BUCKET
|
||||
s3_client().head_bucket(Bucket=BUCKET)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Explicit, allowlisted OTLP instrumentation; no automatic headers, SQL or bodies.
|
||||
SDK exporters run asynchronously with bounded queues; telemetry is best effort.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from pathlib import Path
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
|
||||
PROP = TraceContextTextMapPropagator()
|
||||
tracer = trace.get_tracer('messenger.lab6')
|
||||
log = logging.getLogger('messenger.telemetry')
|
||||
log.setLevel(logging.INFO)
|
||||
log.propagate = False
|
||||
providers = []
|
||||
started = False
|
||||
m = None
|
||||
instruments = {}
|
||||
worker_busy = 0
|
||||
backlog_depth = None
|
||||
backlog_age = None
|
||||
broker_ready = None
|
||||
broker_consumers = None
|
||||
QUEUE = 'attachment-processing.v1'
|
||||
BUCKETS = (0.005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 15, 30, 60, 120, 300, 600)
|
||||
|
||||
|
||||
def init(role, *, child=False):
|
||||
global started, tracer, m
|
||||
if started or os.getenv('OTEL_ENABLED', 'no') != 'yes':
|
||||
return
|
||||
started = True
|
||||
resource = Resource.create({'service.name': 'messenger-' + role,
|
||||
'service.version': '1.6.0', 'service.instance.id': os.getenv('API_INSTANCE_ID', socket.gethostname()),
|
||||
'deployment.environment.name': 'lab'})
|
||||
base = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
|
||||
tp = TracerProvider(resource=resource)
|
||||
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=base+'/v1/traces', timeout=2),
|
||||
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
|
||||
trace.set_tracer_provider(tp)
|
||||
tracer = tp.get_tracer('messenger.lab6')
|
||||
lp = LoggerProvider(resource=resource)
|
||||
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=base+'/v1/logs', timeout=2),
|
||||
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
|
||||
log.addHandler(LoggingHandler(logger_provider=lp))
|
||||
providers.extend([tp, lp])
|
||||
if child:
|
||||
return
|
||||
reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=base+'/v1/metrics', timeout=2),
|
||||
export_interval_millis=3000, export_timeout_millis=2000)
|
||||
mp = MeterProvider(resource=resource, metric_readers=[reader], views=[View(instrument_name=n,
|
||||
aggregation=ExplicitBucketHistogramAggregation(boundaries=BUCKETS)) for n in
|
||||
('lab_http_duration','lab_job_duration','lab_ready_duration')])
|
||||
metrics.set_meter_provider(mp)
|
||||
providers.append(mp)
|
||||
m = mp.get_meter('messenger.lab6')
|
||||
for name in ('lab_http_requests','lab_jobs','lab_job_retries'):
|
||||
instruments[name] = m.create_counter(name)
|
||||
for name in ('lab_http_duration','lab_job_duration','lab_ready_duration'):
|
||||
instruments[name] = m.create_histogram(name, unit='s')
|
||||
if role == 'api':
|
||||
for name, key in [('lab_db_pool_used','used'),('lab_db_pool_waiting','requests_waiting'),('lab_db_pool_capacity','pool_max')]:
|
||||
m.create_observable_gauge(name, callbacks=[lambda _, key=key: pool_stats(key)])
|
||||
if role == 'worker':
|
||||
m.create_observable_gauge('lab_worker_busy', callbacks=[lambda _: [metrics.Observation(worker_busy)]])
|
||||
m.create_observable_gauge('lab_worker_capacity', callbacks=[lambda _: [metrics.Observation(1)]])
|
||||
if role == 'dispatcher':
|
||||
m.create_observable_gauge('lab_broker_ready', callbacks=[lambda _: queue_observation(broker_ready)])
|
||||
m.create_observable_gauge('lab_broker_consumers', callbacks=[lambda _: queue_observation(broker_consumers)])
|
||||
m.create_observable_gauge('lab_queue_depth', callbacks=[lambda _: queue_observation(backlog_depth)])
|
||||
m.create_observable_gauge('lab_queue_oldest_age', unit='s', callbacks=[lambda _: queue_observation(backlog_age)])
|
||||
m.create_observable_gauge('lab_container_memory', unit='By', callbacks=[lambda _: container_observation('memory')])
|
||||
m.create_observable_gauge('lab_container_cpu_cores', callbacks=[lambda _: container_observation('cpu')])
|
||||
|
||||
|
||||
def pool_stats(key):
|
||||
from . import storage
|
||||
if storage.pool is None:
|
||||
return []
|
||||
stats = storage.pool.get_stats()
|
||||
val = stats.get('pool_size',0)-stats.get('pool_available',0) if key=='used' else stats.get(key,0)
|
||||
return [metrics.Observation(max(0,val))]
|
||||
|
||||
|
||||
def queue_observation(value):
|
||||
return [] if value is None else [metrics.Observation(value, {'queue': QUEUE})]
|
||||
|
||||
|
||||
_cpu_previous = None
|
||||
|
||||
def container_observation(kind):
|
||||
global _cpu_previous
|
||||
try:
|
||||
if kind == 'memory':
|
||||
return [metrics.Observation(int(Path('/sys/fs/cgroup/memory.current').read_text()))]
|
||||
cpu = int(dict(line.split() for line in Path('/sys/fs/cgroup/cpu.stat').read_text().splitlines())['usage_usec']) / 1e6
|
||||
now = time.monotonic()
|
||||
old = _cpu_previous
|
||||
_cpu_previous = (now,cpu)
|
||||
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
|
||||
except (OSError, KeyError, ValueError):
|
||||
# cgroup v1 fallback, process tree (not full container accounting).
|
||||
import psutil
|
||||
try:
|
||||
p = psutil.Process()
|
||||
processes = [p]+p.children(recursive=True)
|
||||
if kind == 'memory':
|
||||
value = 0
|
||||
for proc in processes:
|
||||
try: value += proc.memory_info().rss
|
||||
except psutil.Error: pass
|
||||
return [metrics.Observation(value)]
|
||||
# cgroup v1 (Docker Linux) if available, CPU usage in nanoseconds.
|
||||
for path in ('/sys/fs/cgroup/cpuacct/cpuacct.usage','/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage'):
|
||||
if Path(path).is_file():
|
||||
cpu = int(Path(path).read_text()) / 1e9
|
||||
now = time.monotonic()
|
||||
old = _cpu_previous
|
||||
_cpu_previous = (now,cpu)
|
||||
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
|
||||
except (psutil.Error,OSError,ValueError):
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def count(name, attrs=None):
|
||||
if name in instruments: instruments[name].add(1, attrs or {})
|
||||
|
||||
|
||||
def observe(name, value, attrs=None):
|
||||
if name in instruments: instruments[name].record(max(0,value), attrs or {})
|
||||
|
||||
|
||||
def event(name, *, level=logging.INFO, **attrs):
|
||||
# Callers pass fixed event names and technical ID/status only, never payloads.
|
||||
if started:
|
||||
log.log(level, name, extra={'event':name, **attrs})
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name, *, kind=SpanKind.INTERNAL, context=None, attrs=None):
|
||||
with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attrs,
|
||||
record_exception=False, set_status_on_exception=False) as s:
|
||||
try:
|
||||
yield s
|
||||
except BaseException as exc:
|
||||
s.set_status(Status(StatusCode.ERROR))
|
||||
s.set_attribute('error.type', type(exc).__name__)
|
||||
event('operation.error', level=logging.ERROR, operation=name, error_type=type(exc).__name__)
|
||||
raise
|
||||
|
||||
|
||||
def inject():
|
||||
result = {}
|
||||
PROP.inject(result)
|
||||
return result
|
||||
|
||||
|
||||
def extract(carrier):
|
||||
# Ignore baggage and all unrelated fields.
|
||||
return PROP.extract({k:v for k,v in carrier.items() if k in ('traceparent','tracestate') and isinstance(v,str)})
|
||||
|
||||
|
||||
def dependency_span(name, **kwargs):
|
||||
return span(name, **kwargs) if trace_id() else nullcontext()
|
||||
|
||||
|
||||
def trace_id():
|
||||
ctx = trace.get_current_span().get_span_context()
|
||||
return f'{ctx.trace_id:032x}' if ctx.is_valid else ''
|
||||
|
||||
|
||||
def flush():
|
||||
for p in providers:
|
||||
p.force_flush(timeout_millis=2500)
|
||||
|
||||
|
||||
class S3Tracing:
|
||||
"""Only operation name is exported; bucket/key/bytes/signed URLs never exported."""
|
||||
def __init__(self, client): self.client = client
|
||||
def __getattr__(self, name):
|
||||
f = getattr(self.client,name)
|
||||
if name not in ('upload_fileobj','get_object','put_object','copy_object','delete_object','head_object','head_bucket'):
|
||||
return f
|
||||
def call(*a, **kw):
|
||||
with dependency_span('s3.'+name, kind=SpanKind.CLIENT, attrs={'storage.system':'s3'}):
|
||||
return f(*a, **kw)
|
||||
return call
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.10-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-lab}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||
volumes: ["postgres_data:/var/lib/postgresql/data"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
restart: unless-stopped
|
||||
networks: [storage]
|
||||
|
||||
redis:
|
||||
image: redis:7.4.5-alpine
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", "60", "1"]
|
||||
volumes: ["redis_data:/data"]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
restart: unless-stopped
|
||||
networks: [storage]
|
||||
|
||||
broker:
|
||||
image: rabbitmq:4.1.8-management-alpine
|
||||
hostname: lab6-broker
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-labworker}
|
||||
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Set BROKER_PASSWORD in .env}
|
||||
volumes: ["rabbitmq_data:/var/lib/rabbitmq"]
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 25
|
||||
start_period: 15s
|
||||
mem_limit: 512m
|
||||
cpus: 0.75
|
||||
restart: unless-stopped
|
||||
networks: [storage]
|
||||
|
||||
s3-policy:
|
||||
build: .
|
||||
user: "0:0"
|
||||
command: ["python", "-m", "app.s3_policy"]
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:?Set S3_ADMIN_ACCESS_KEY}
|
||||
S3_ADMIN_SECRET_KEY: ${S3_ADMIN_SECRET_KEY:?Set S3_ADMIN_SECRET_KEY}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
|
||||
volumes: ["s3_config:/s3-config"]
|
||||
restart: "no"
|
||||
networks: [storage]
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.53
|
||||
command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.port=8333", "-s3.iam=false", "-s3.config=/s3-config/config.json"]
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- s3_config:/s3-config:ro
|
||||
- ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro
|
||||
depends_on:
|
||||
s3-policy:
|
||||
condition: service_completed_successfully
|
||||
expose: ["8333"]
|
||||
mem_limit: 640m
|
||||
cpus: 1.0
|
||||
restart: unless-stopped
|
||||
networks: [storage]
|
||||
|
||||
s3-bootstrap:
|
||||
build: .
|
||||
command: ["python", "-m", "app.s3_bootstrap"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:?Set S3_ADMIN_ACCESS_KEY}
|
||||
S3_ADMIN_SECRET_KEY: ${S3_ADMIN_SECRET_KEY:?Set S3_ADMIN_SECRET_KEY}
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_started
|
||||
restart: "no"
|
||||
networks: [storage]
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
command: ["python", "-m", "app.migrate"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
networks: [storage]
|
||||
|
||||
reconcile:
|
||||
profiles: [maintenance]
|
||||
build: .
|
||||
command: ["python", "-m", "app.reconcile"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
restart: "no"
|
||||
networks: [storage]
|
||||
|
||||
dispatcher:
|
||||
build: .
|
||||
command: ["python", "-m", "app.jobs", "dispatch"]
|
||||
environment: &job_env
|
||||
OTEL_ENABLED: ${OTEL_ENABLED:-yes}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
|
||||
BROKER_HOST: broker
|
||||
BROKER_USER: ${BROKER_USER:-labworker}
|
||||
BROKER_PASSWORD: ${BROKER_PASSWORD:?Set BROKER_PASSWORD in .env}
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
depends_on:
|
||||
broker:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
mem_limit: 192m
|
||||
cpus: 0.5
|
||||
restart: unless-stopped
|
||||
networks: [storage, telemetry]
|
||||
|
||||
worker:
|
||||
build: .
|
||||
command: ["python", "-m", "app.jobs", "worker"]
|
||||
environment:
|
||||
<<: *job_env
|
||||
LAB6_FAULT_RETRY_ONCE: ${LAB6_FAULT_RETRY_ONCE:-off}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||||
LAB5_FAULT_AFTER_S3: ${LAB5_FAULT_AFTER_S3:-off}
|
||||
depends_on:
|
||||
broker:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
mem_limit: 768m
|
||||
cpus: 1.0
|
||||
stop_grace_period: 75s
|
||||
restart: unless-stopped
|
||||
networks: [storage, telemetry]
|
||||
|
||||
api1:
|
||||
build: .
|
||||
environment: &api_env
|
||||
OTEL_ENABLED: ${OTEL_ENABLED:-yes}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
APP_PORT: "8080"
|
||||
API_INSTANCE_ID: api1
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-8}
|
||||
JOB_BACKLOG_LIMIT: ${JOB_BACKLOG_LIMIT:-1000}
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
|
||||
depends_on: &api_depends
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
expose: ["8080"]
|
||||
healthcheck: &api_health
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 15
|
||||
start_period: 10s
|
||||
mem_limit: 384m
|
||||
cpus: 0.75
|
||||
stop_grace_period: 12s
|
||||
restart: unless-stopped
|
||||
networks: [frontend, storage, telemetry]
|
||||
|
||||
api2:
|
||||
build: .
|
||||
environment:
|
||||
<<: *api_env
|
||||
API_INSTANCE_ID: api2
|
||||
depends_on: *api_depends
|
||||
expose: ["8080"]
|
||||
healthcheck: *api_health
|
||||
mem_limit: 384m
|
||||
cpus: 0.75
|
||||
stop_grace_period: 12s
|
||||
restart: unless-stopped
|
||||
networks: [frontend, storage, telemetry]
|
||||
|
||||
lb:
|
||||
image: haproxy:3.1.7-alpine
|
||||
volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"]
|
||||
ports: ["127.0.0.1:8080:8080"]
|
||||
depends_on:
|
||||
api1:
|
||||
condition: service_healthy
|
||||
api2:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
restart: unless-stopped
|
||||
networks: [frontend]
|
||||
|
||||
telemetry-init:
|
||||
image: busybox:1.37.0
|
||||
user: "0:0"
|
||||
command: ["sh", "-ec", "chown 10001:10001 /tempo /loki; chown 65534:65534 /prometheus; chown 472:472 /grafana"]
|
||||
volumes:
|
||||
- tempo_data:/tempo
|
||||
- loki_data:/loki
|
||||
- prometheus_data:/prometheus
|
||||
- grafana_data:/grafana
|
||||
network_mode: none
|
||||
restart: "no"
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.123.0
|
||||
command: ["--config=/etc/otelcol/config.yaml"]
|
||||
volumes: ["./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro"]
|
||||
environment:
|
||||
GOMEMLIMIT: 180MiB
|
||||
mem_limit: 256m
|
||||
cpus: 0.75
|
||||
restart: unless-stopped
|
||||
networks: [telemetry]
|
||||
|
||||
prometheus:
|
||||
depends_on:
|
||||
telemetry-init:
|
||||
condition: service_completed_successfully
|
||||
image: prom/prometheus:v3.2.1
|
||||
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=72h", "--storage.tsdb.retention.size=512MB"]
|
||||
volumes:
|
||||
- ./infra/otel/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/otel/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports: ["127.0.0.1:9090:9090"]
|
||||
mem_limit: 384m
|
||||
restart: unless-stopped
|
||||
networks: [telemetry]
|
||||
|
||||
loki:
|
||||
depends_on:
|
||||
telemetry-init:
|
||||
condition: service_completed_successfully
|
||||
image: grafana/loki:3.4.2
|
||||
command: ["-config.file=/etc/loki/local-config.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/loki.yaml:/etc/loki/local-config.yaml:ro
|
||||
- loki_data:/loki
|
||||
ports: ["127.0.0.1:3100:3100"]
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
networks: [telemetry]
|
||||
|
||||
tempo:
|
||||
depends_on:
|
||||
telemetry-init:
|
||||
condition: service_completed_successfully
|
||||
image: grafana/tempo:2.7.2
|
||||
command: ["-config.file=/etc/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/tempo.yaml:/etc/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
ports: ["127.0.0.1:3200:3200"]
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
networks: [telemetry]
|
||||
|
||||
grafana:
|
||||
depends_on:
|
||||
telemetry-init:
|
||||
condition: service_completed_successfully
|
||||
image: grafana/grafana:11.6.0
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
ports: ["127.0.0.1:3000:3000"]
|
||||
mem_limit: 384m
|
||||
restart: unless-stopped
|
||||
networks: [telemetry]
|
||||
|
||||
networks:
|
||||
telemetry:
|
||||
frontend:
|
||||
storage:
|
||||
internal: true
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
s3_config:
|
||||
rabbitmq_data:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
@@ -0,0 +1,33 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username VARCHAR(32) NOT NULL UNIQUE,
|
||||
display_name BYTEA NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT users_display_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title BYTEA NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
user_id UUID NOT NULL REFERENCES users(id),
|
||||
position INTEGER NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_members_user ON chat_members (user_id, chat_id);
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
sender_id UUID NOT NULL,
|
||||
text BYTEA NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id),
|
||||
CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_history ON messages (chat_id, created_at, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_chats_history ON chats (created_at, id);
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Same transaction creates message and idempotency receipt. Unique key arbitrates concurrent replicas.
|
||||
CREATE TABLE IF NOT EXISTS message_idempotency (
|
||||
actor_id UUID NOT NULL REFERENCES users(id),
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
method VARCHAR(8) NOT NULL DEFAULT 'POST',
|
||||
idem_key VARCHAR(128) NOT NULL,
|
||||
body_sha256 CHAR(64) NOT NULL,
|
||||
message_id UUID NOT NULL REFERENCES messages(id),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (actor_id, chat_id, method, idem_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_idempotency_age ON message_idempotency (created_at);
|
||||
@@ -0,0 +1,25 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_bytes;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000);
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
owner_id UUID NOT NULL REFERENCES users(id),
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
backup_key TEXT NOT NULL UNIQUE,
|
||||
filename TEXT NOT NULL,
|
||||
content_type VARCHAR(32) NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes BIGINT NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id);
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id),
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
position INTEGER NOT NULL CHECK(position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id,position)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_message_attachments_attachment ON message_attachments(attachment_id);
|
||||
@@ -0,0 +1,58 @@
|
||||
-- Lab5: stage durably, then transact job and outbox atomically.
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'ready'
|
||||
CHECK (status IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code TEXT;
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT;
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INTEGER NOT NULL DEFAULT 1;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_attachments_staging ON attachments(staging_key) WHERE staging_key IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachment_variants (
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1),
|
||||
name TEXT NOT NULL CHECK (name IN ('thumbnail','crop','optimized')),
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL CHECK (size_bytes > 0),
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
width INTEGER NOT NULL CHECK (width > 0),
|
||||
height INTEGER NOT NULL CHECK (height > 0),
|
||||
PRIMARY KEY (attachment_id,pipeline_version,name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS processing_jobs (
|
||||
id UUID PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL UNIQUE REFERENCES attachments(id),
|
||||
pipeline_version INTEGER NOT NULL DEFAULT 1,
|
||||
status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued','processing','ready','failed')),
|
||||
attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||||
next_run_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
lease_token UUID,
|
||||
leased_until TIMESTAMPTZ,
|
||||
error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_leases ON processing_jobs(status, leased_until);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS job_outbox (
|
||||
id UUID PRIMARY KEY,
|
||||
job_id UUID NOT NULL REFERENCES processing_jobs(id),
|
||||
payload JSONB NOT NULL,
|
||||
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
published_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_outbox_due ON job_outbox(available_at, created_at) WHERE published_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS job_replay_audit (
|
||||
id UUID PRIMARY KEY,
|
||||
job_id UUID NOT NULL REFERENCES processing_jobs(id),
|
||||
replayed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
action TEXT NOT NULL DEFAULT 'operator_replay'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS job_fault_once (
|
||||
job_id UUID NOT NULL REFERENCES processing_jobs(id),
|
||||
fault TEXT NOT NULL,
|
||||
PRIMARY KEY (job_id,fault)
|
||||
);
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"concurrency": 10,
|
||||
"accepted_uploads": 10,
|
||||
"ready_attachments": 10,
|
||||
"failed_requests": 0,
|
||||
"acceptance_elapsed_seconds": 2.515,
|
||||
"acceptance_rps": 3.976,
|
||||
"upload_latency_p50_ms": 2296.5,
|
||||
"upload_latency_max_ms": 2484.0,
|
||||
"all_ready_elapsed_seconds": 36.406,
|
||||
"bytes_per_file": 111817,
|
||||
"note": "HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats."
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 113 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 79 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 150 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 111 KiB |
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"scenario": "backend-down",
|
||||
"status": "PASS",
|
||||
"result": {
|
||||
"successful_messages_during_tempo_outage": 16,
|
||||
"trace_id": "9aa3b3a670850e443df5779f0f60d8a8",
|
||||
"max_sampled_collector_rss_bytes": 210169856.0,
|
||||
"collector_memory_limit_bytes": 268435456,
|
||||
"rss_sample_count": 17,
|
||||
"max_export_queue": 18.0,
|
||||
"export_queue_capacity": 64.0,
|
||||
"failed_span_counter_delta": 21.0,
|
||||
"restart_state": "0 false",
|
||||
"recovered_trace_id": "89192e038ce76d9f10bc2d714af7f77d"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"scenario": "db-error",
|
||||
"status": "PASS",
|
||||
"result": {
|
||||
"trace_id": "859975662df88e09af421eb97ce16911",
|
||||
"request_id": "174f09c4345a425d91854d9d614fc1ca",
|
||||
"http_codes": [
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503,
|
||||
503
|
||||
],
|
||||
"firing_error_ratio": 1.0,
|
||||
"resolved_error_ratio": 0.0,
|
||||
"error_spans": [
|
||||
{
|
||||
"name": "db.transaction",
|
||||
"error_type": "QueryCanceled"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"scenario": "history",
|
||||
"status": "PASS",
|
||||
"result": {
|
||||
"trace_id": "24926a7aa71429d9947f0c17f08b63a2",
|
||||
"historical_query_timestamp": 1794929612.5405939,
|
||||
"metric_before": 148.0,
|
||||
"metric_after": 148.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,532 @@
|
||||
{
|
||||
"scenario": "metrics",
|
||||
"status": "PASS",
|
||||
"result": {
|
||||
"api_rps": {
|
||||
"query": "sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s]))",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1794929001.859,
|
||||
"0.4074074074074075"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"error_ratio": {
|
||||
"query": "(sum(rate(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s])),0.001)",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1794929001.864,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"p50": {
|
||||
"query": "histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])))",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1794929001.88,
|
||||
"0.017503056800918893"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"p95": {
|
||||
"query": "histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])))",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1794929001.891,
|
||||
"0.0725"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"p99": {
|
||||
"query": "histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])))",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1794929001.897,
|
||||
"0.09450000000000001"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"jobs": {
|
||||
"query": "lab_jobs_total",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_jobs_total",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "9d2e6ea768ae",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"job_type": "attachment.process.v1",
|
||||
"outcome": "ready",
|
||||
"service_instance_id": "9d2e6ea768ae",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929001.904,
|
||||
"2"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"job_histogram": {
|
||||
"query": "lab_job_duration_seconds_count",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_job_duration_seconds_count",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "9d2e6ea768ae",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"job_type": "attachment.process.v1",
|
||||
"outcome": "ready",
|
||||
"service_instance_id": "9d2e6ea768ae",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929001.927,
|
||||
"2"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"ready_histogram": {
|
||||
"query": "lab_ready_duration_seconds_count",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_ready_duration_seconds_count",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "9d2e6ea768ae",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"job_type": "attachment.process.v1",
|
||||
"service_instance_id": "9d2e6ea768ae",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929001.932,
|
||||
"2"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"depth": {
|
||||
"query": "max(lab_queue_depth)",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1794929001.937,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"oldest": {
|
||||
"query": "max(lab_queue_oldest_age_seconds)",
|
||||
"result": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1794929001.957,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"broker_ready": {
|
||||
"query": "lab_broker_ready",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_broker_ready",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "23fa9474ac6a",
|
||||
"exported_job": "messenger-dispatcher",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"queue": "attachment-processing.v1",
|
||||
"service_instance_id": "23fa9474ac6a",
|
||||
"service_name": "messenger-dispatcher",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929001.973,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"broker_consumers": {
|
||||
"query": "lab_broker_consumers",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_broker_consumers",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "23fa9474ac6a",
|
||||
"exported_job": "messenger-dispatcher",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"queue": "attachment-processing.v1",
|
||||
"service_instance_id": "23fa9474ac6a",
|
||||
"service_name": "messenger-dispatcher",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929001.988,
|
||||
"1"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"pool_used": {
|
||||
"query": "lab_db_pool_used",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_db_pool_used",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "api1",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "api1",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929001.993,
|
||||
"0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_db_pool_used",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "api2",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "api2",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929001.993,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"pool_waiting": {
|
||||
"query": "lab_db_pool_waiting",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_db_pool_waiting",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "api1",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "api1",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.019,
|
||||
"0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_db_pool_waiting",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "api2",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "api2",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.019,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"capacity": {
|
||||
"query": "lab_worker_capacity",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_worker_capacity",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "9d2e6ea768ae",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "9d2e6ea768ae",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.036,
|
||||
"1"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"busy": {
|
||||
"query": "lab_worker_busy",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_worker_busy",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "9d2e6ea768ae",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "9d2e6ea768ae",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.067,
|
||||
"0"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"cpu": {
|
||||
"query": "lab_container_cpu_cores",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_container_cpu_cores",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "9d2e6ea768ae",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "9d2e6ea768ae",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.098,
|
||||
"0.0012147006450733576"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_container_cpu_cores",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "api1",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "api1",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.098,
|
||||
"0.1248036553953329"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_container_cpu_cores",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "api2",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "api2",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.098,
|
||||
"0.1264468449809642"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_container_cpu_cores",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "23fa9474ac6a",
|
||||
"exported_job": "messenger-dispatcher",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "23fa9474ac6a",
|
||||
"service_name": "messenger-dispatcher",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.098,
|
||||
"0.009300433365947897"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"memory": {
|
||||
"query": "lab_container_memory_bytes",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_container_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "9d2e6ea768ae",
|
||||
"exported_job": "messenger-worker",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "9d2e6ea768ae",
|
||||
"service_name": "messenger-worker",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.107,
|
||||
"104075264"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_container_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "api1",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "api1",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.107,
|
||||
"152047616"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_container_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "api2",
|
||||
"exported_job": "messenger-api",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "api2",
|
||||
"service_name": "messenger-api",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.107,
|
||||
"144863232"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "lab_container_memory_bytes",
|
||||
"deployment_environment_name": "lab",
|
||||
"exported_instance": "23fa9474ac6a",
|
||||
"exported_job": "messenger-dispatcher",
|
||||
"instance": "otel-collector:8889",
|
||||
"job": "lab-otlp",
|
||||
"service_instance_id": "23fa9474ac6a",
|
||||
"service_name": "messenger-dispatcher",
|
||||
"service_version": "1.6.0",
|
||||
"telemetry_sdk_language": "python",
|
||||
"telemetry_sdk_name": "opentelemetry",
|
||||
"telemetry_sdk_version": "1.42.1"
|
||||
},
|
||||
"value": [
|
||||
1794929002.107,
|
||||
"104079360"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"scenario": "queue-lag",
|
||||
"status": "PASS",
|
||||
"result": {
|
||||
"trace_id": "6bf0ddf7b17bb3e846aa931aab6e97ef",
|
||||
"firing_lag_seconds": 27.574387,
|
||||
"firing_depth": 1.0,
|
||||
"detected_seconds": 34.235,
|
||||
"resolved_depth": 0.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
{
|
||||
"scenario": "trace-retry",
|
||||
"status": "PASS",
|
||||
"result": {
|
||||
"trace_id": "4d8711b71965876a59aa5594ad0d51a0",
|
||||
"request_id": "d92704875e2b4f0cb3b76f5354943a99",
|
||||
"spans": [
|
||||
{
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"id": "k4uo/LzSEq8=",
|
||||
"parent": null,
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.transaction",
|
||||
"id": "DzqQmZ0ugps=",
|
||||
"parent": "k4uo/LzSEq8=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.transaction",
|
||||
"id": "2ChmWodQe24=",
|
||||
"parent": "k4uo/LzSEq8=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "s3.upload_fileobj",
|
||||
"id": "Vy3PofMYPBU=",
|
||||
"parent": "k4uo/LzSEq8=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.transaction",
|
||||
"id": "BmV9vHb3CTw=",
|
||||
"parent": "k4uo/LzSEq8=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.publish",
|
||||
"id": "eRBOkrMeoa8=",
|
||||
"parent": "BmV9vHb3CTw=",
|
||||
"attempt": "1",
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.consume",
|
||||
"id": "8iVPBhXuImY=",
|
||||
"parent": "eRBOkrMeoa8=",
|
||||
"attempt": "1",
|
||||
"outcome": "retry"
|
||||
},
|
||||
{
|
||||
"name": "db.job.transaction",
|
||||
"id": "lZp3w62oH14=",
|
||||
"parent": "8iVPBhXuImY=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.job.transaction",
|
||||
"id": "TO5Q0Nf/ECg=",
|
||||
"parent": "8iVPBhXuImY=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.job.transaction",
|
||||
"id": "aWYPqHrrvag=",
|
||||
"parent": "8iVPBhXuImY=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.publish",
|
||||
"id": "WBujc7E21qk=",
|
||||
"parent": "aWYPqHrrvag=",
|
||||
"attempt": "2",
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.consume",
|
||||
"id": "jvmmPa040qY=",
|
||||
"parent": "WBujc7E21qk=",
|
||||
"attempt": "2",
|
||||
"outcome": "ready"
|
||||
},
|
||||
{
|
||||
"name": "db.job.transaction",
|
||||
"id": "gRcYatAJcDI=",
|
||||
"parent": "jvmmPa040qY=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "s3.delete_object",
|
||||
"id": "5apnWPa0LHs=",
|
||||
"parent": "jvmmPa040qY=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.job.transaction",
|
||||
"id": "GHNw+OD6g+E=",
|
||||
"parent": "jvmmPa040qY=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.job.transaction",
|
||||
"id": "t6ytfExz/wY=",
|
||||
"parent": "jvmmPa040qY=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.job.transaction",
|
||||
"id": "nGilf7vU83w=",
|
||||
"parent": "jvmmPa040qY=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "image.subprocess",
|
||||
"id": "30CroshuABo=",
|
||||
"parent": "jvmmPa040qY=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.publish_result",
|
||||
"id": "FhqMuTZIy5E=",
|
||||
"parent": "jvmmPa040qY=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.job.transaction",
|
||||
"id": "8it2Knjruyc=",
|
||||
"parent": "FhqMuTZIy5E=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "image.transform",
|
||||
"id": "NmBuL1p0oMs=",
|
||||
"parent": "30CroshuABo=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "db.load_staging_metadata",
|
||||
"id": "sDN1N6qO6A8=",
|
||||
"parent": "NmBuL1p0oMs=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "s3.get_object",
|
||||
"id": "mf9t+iylMEA=",
|
||||
"parent": "NmBuL1p0oMs=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "s3.put_object",
|
||||
"id": "AOPN3aX8c/k=",
|
||||
"parent": "NmBuL1p0oMs=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "s3.put_object",
|
||||
"id": "yjZiBqAz1v0=",
|
||||
"parent": "NmBuL1p0oMs=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "s3.put_object",
|
||||
"id": "9kNHAG8kgW4=",
|
||||
"parent": "NmBuL1p0oMs=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "s3.copy_object",
|
||||
"id": "hRIkPPRqeho=",
|
||||
"parent": "NmBuL1p0oMs=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
},
|
||||
{
|
||||
"name": "s3.copy_object",
|
||||
"id": "zhrE/ErGjRE=",
|
||||
"parent": "NmBuL1p0oMs=",
|
||||
"attempt": null,
|
||||
"outcome": null
|
||||
}
|
||||
],
|
||||
"worker_log_streams": 5,
|
||||
"worker_events": [
|
||||
"image.transformed",
|
||||
"job.claimed",
|
||||
"job.ready",
|
||||
"job.retry"
|
||||
],
|
||||
"retry_chain": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
# Локальные результаты подготовки Lab6 (2026-11-17)
|
||||
|
||||
Не являются результатами Docker-интеграции. Среда подготовки: Linux/Python 3.13, установлен SDK OpenTelemetry 1.42.1, Docker CLI отсутствует. Файлы исходного задания проверены byte-for-byte: изменены только .env.example, Makefile (python вместо python3 для Windows) и REPORT.md. Исходные публичные проверки, CI, job schema, OpenAPI, README и PR template сохранены.
|
||||
|
||||
```text
|
||||
make check
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python scripts/check_lab6.py
|
||||
PASS: 65 literal SQL bindings; pinned Compose images, mounts and networks; 3 OTLP pipelines, dashboards and alert rules (static only)
|
||||
python tests/jobs_sql_contract.py
|
||||
PASS: attachment_variants INSERT has 9 columns, 9 placeholders, 9 arguments
|
||||
python tests/own_unit.py
|
||||
PASS: models, cursor scoping, limits, Unicode U+0000
|
||||
python tests/telemetry_unit.py
|
||||
PASS: real OTLP trace/log/metric encoding, parent links, histogram buckets, safe S3 attributes
|
||||
```
|
||||
|
||||
`telemetry_unit.py` использует настоящий Python SDK и локальный HTTP OTLP receiver, декодирует protobuf и проверяет parent IDs, trace/span IDs в логе и histogram boundary values; не заменяет проверку Collector, Loki, Tempo, Prometheus, Grafana. Дополнительно: все Python modules успешно разобраны AST; JSON Schema examples и job с traceparent прошли jsonschema validation. Full OpenAPI validator отсутствует в среде подготовки; полный `make validate` не исполнялся. Точная конфигурация/реальные команды backend проверяются на стенде при запуске. Бизнес-сценарии, backend failure, alert firing/resolved, telemetry persistence и нагрузка на Lab6 ещё не подтверждены.
|
||||
@@ -0,0 +1,90 @@
|
||||
# Подтверждённые результаты Lab6 — 17.11.2026
|
||||
|
||||
Целевая оценка: 4. Стенд: Windows, PowerShell, Docker Desktop (Linux containers). Источник — фактический вывод команд и четыре оригинальных PNG из Grafana, предоставленные после запуска Lab6. Это сводка наблюдений, не повторный прогон тестов. Версия commit и параметры оборудования не зафиксированы. Секреты не включены.
|
||||
|
||||
## Запуск и проверка конфигурации
|
||||
|
||||
| Команда | Результат |
|
||||
| --- | --- |
|
||||
| `docker compose --env-file .env config --quiet` | PASS, ошибки отсутствовали |
|
||||
| `docker compose --env-file .env run --rm --no-deps otel-collector validate --config=/etc/otelcol/config.yaml` | PASS, ошибок в выводе нет |
|
||||
| `docker compose --env-file .env run --rm --no-deps --entrypoint /bin/promtool prometheus check config /etc/prometheus/prometheus.yml` | SUCCESS: конфигурация валидна, 2 правила алертов |
|
||||
| `docker compose --env-file .env up -d --build --wait` и `ps` | Сборка завершена; 14 основных контейнеров Up; API1/API2, broker, lb, PostgreSQL, Redis healthy |
|
||||
| `make check` | PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. |
|
||||
| `make test` | PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. |
|
||||
|
||||
## Наблюдаемость: реальные сценарии
|
||||
|
||||
Перед fault-тестами: `$env:LAB6_ALLOW_FAILURE_TEST = "yes"`.
|
||||
|
||||
| Команда (`python tests/observability_live.py --scenario ...`) | Наблюдаемый результат | Файл на стенде |
|
||||
| --- | --- | --- |
|
||||
| `metrics` | PASS: все ожидаемые ряды существуют, запрещённых ID labels нет | `lab6-metrics.json` |
|
||||
| `trace-retry` | PASS: API/producer/consumer/S3/DB, retry и связанные логи | `lab6-trace-retry.json` |
|
||||
| `queue-lag` | PASS: alert firing, oldest age 27.574387 s, queued=1. После восстановления worker: ready и resolved | `lab6-queue-lag.json` |
|
||||
| `db-error` | PASS: DB lock → QueryCanceled/503, error ratio 1.0, alert firing → resolved | `lab6-db-error.json` |
|
||||
| `backend-down` | PASS после исправления диагностического теста: 16 записей и обработка изображения во время отказа Tempo; ошибки экспорта видимы; RSS/очередь ограничены; свежая трасса восстановилась | `lab6-backend-down.json` |
|
||||
| `history` | PASS: ранее сохранённые метрики, логи, трассы доступны после перезапуска backend | `lab6-history.json` |
|
||||
|
||||
Сквозная трасса: `4d8711b71965876a59aa5594ad0d51a0`. Может быть удалена политикой retention; при необходимости повторить `trace-retry`.
|
||||
|
||||
Первый запуск `backend-down` дал `FAIL: Collector RSS absent or above container limit`. Применён `lab6_backend_down_fix.zip` (изменение `tests/observability_live.py`, не бизнес-кода). Исправленный тест принимает нормализованные имена RSS, различает отсутствующие метрики и реальные превышения, измеряет память/очередь и проверяет состояние контейнера. Подтверждённый повторный вывод:
|
||||
|
||||
```text
|
||||
Collector baseline: RSS=197.35 MiB, queue=0/64, limit=256 MiB
|
||||
Collector outage: sampled RSS max=200.43 MiB, queue max=18/64, successful writes=16
|
||||
PASS backend-down: writes and image processing persisted, telemetry failed visibly; queue/RAM bounded, fresh trace recovered
|
||||
Saved evidence\lab6-backend-down.json
|
||||
```
|
||||
|
||||
RSS — наблюдаемый максимум выборки, не абсолютный пик. 64 — вместимость очереди конкретного exporter, а не размер RabbitMQ. Успешная работа при одном коротком отказе не гарантирует отсутствие потерь телеметрии при произвольной длительности отказа. Числовую дельту failed exports следует брать из оригинального JSON на стенде.
|
||||
|
||||
## Регрессия Lab5 на стенде Lab6
|
||||
|
||||
| Команда | Результат |
|
||||
| --- | --- |
|
||||
| `python tests/own_unit.py` | PASS: models, cursor scoping, limits, Unicode U+0000 |
|
||||
| `python tests/attachments_negative.py` | PASS: MIME, размер, имя файла, ACL, варианты и cross-chat |
|
||||
| `python tests/idempotency_parallel.py` | PASS: две API-реплики, одно сообщение, replay 201, conflicting body 409 |
|
||||
| `python tests/pipeline_resilience.py --scenario paused` | PASS: durable queued → ready, оригинал и 3 варианта, без дубликатов |
|
||||
| `python tests/pipeline_resilience.py --scenario outbox-crash` | PASS: dispatcher exit 78 после commit outbox, до publish; восстановление queued → ready |
|
||||
| `python tests/pipeline_resilience.py --scenario worker-crash` | PASS: worker exit 77 после S3 effects, до result/ack; восстановление queued → ready, без дубликатов |
|
||||
|
||||
Для трёх fault-тестов использовано `$env:LAB5_ALLOW_FAILURE_TEST = "yes"`.
|
||||
|
||||
## Нагрузка
|
||||
|
||||
`python tests/attachments_concurrent.py --concurrency 10 --output evidence/attachments-concurrent.json`
|
||||
|
||||
| Показатель | Фактическое значение |
|
||||
| --- | ---: |
|
||||
| Параллельных загрузок | 10 |
|
||||
| HTTP 202 принятых | 10 |
|
||||
| Вложений ready | 10 |
|
||||
| Ошибок запроса | 0 |
|
||||
| Вариантов на вложение | 3 |
|
||||
| Размер исходного файла, B | 111817 |
|
||||
| Приём всех запросов, s | 2.515 |
|
||||
| Acceptance RPS | 3.976 |
|
||||
| Upload latency p50, ms | 2296.5 |
|
||||
| Upload latency max, ms | 2484.0 |
|
||||
| Все вложения ready через, s | 36.406 |
|
||||
|
||||
Acceptance RPS не является throughput worker. 36.406 s — общее время до готовности всех десяти вложений, а не latency каждого из них. Данный эксперимент не измеряет предельную пропускную способность; p95/p99 на dashboard относятся к histogram разных наблюдаемых запросов, не вычисляются из p50/max этой таблицы.
|
||||
|
||||
## Grafana: исходные скриншоты
|
||||
|
||||
1. [API throughput, 5xx, latency, jobs](grafana/01-api-and-jobs.png).
|
||||
2. [Worker duration, time to ready, queue depth, oldest age](grafana/02-worker-and-queue.png).
|
||||
3. [DB pool, worker saturation, CPU, RAM](grafana/03-pool-and-resources.png).
|
||||
4. [Collector queue, failed exports, alerts, SLI/SLO](grafana/04-collector-alerts-slo.png).
|
||||
|
||||
По всем 16 панелям видны реальные ряды. При контролируемой деградации видны рост 5xx и latency, накопление очереди и возраст задания, ошибки экспорта Tempo, нарушение цели SLO 99% с последующим восстановлением. Снимки дают визуальное подтверждение, а точные состояния firing/resolved и корреляцию logs/spans подтверждают автоматизированные сценарии выше. При отсутствии job events histogram может не иметь данных на отдельных интервалах — это нормально. CPU измеряется в ядрах, RAM — cgroup usage или RSS fallback согласно названию панели.
|
||||
|
||||
## Границы доказательств и дальнейшие проверки
|
||||
|
||||
- Шесть новых сценариев Lab6 и перечисленные регрессионные проверки подтверждены фактическими результатами. Оценку выставляет преподаватель.
|
||||
- Отдельный closed-loop `tests/load.py`, общие persistence/failover `tests/resilience.py` и `pipeline_extended.py` (broker-persistence, failed-replay, staging-cleanup) для Lab6 не подтверждены предоставленным выводом. Их успехи в Lab5 не перенесены в итоги Lab6.
|
||||
- Не измеряли максимальную длительность отказа telemetry, действительный peak RSS, полный объём потерянных spans и SLA на стороне клиента; счётчики failed exports доступны в данных стенда.
|
||||
- JSON-файлы из таблиц уже созданы на машине автора; эта сводка их не заменяет и не воспроизводит отсутствующие поля.
|
||||
- Для тестирования использовался `.env`, скопированный из `.env.example` без замены значений. Перед новым развёртыванием заменить примерные секреты; действующий `.env` не коммитить. Данные локальны, диагностика доступна на loopback.
|
||||
@@ -0,0 +1,4 @@
|
||||
# Persistent SeaweedFS filer metadata for the single-node lab stack.
|
||||
[leveldb2]
|
||||
enabled = true
|
||||
dir = "/data/filerldb2"
|
||||
@@ -0,0 +1,32 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 1000
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
option dontlognull
|
||||
timeout connect 2s
|
||||
timeout client 60s
|
||||
timeout server 60s
|
||||
timeout http-request 5s
|
||||
|
||||
resolvers docker
|
||||
nameserver localdns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
|
||||
frontend messenger
|
||||
bind *:8080
|
||||
default_backend api_nodes
|
||||
|
||||
backend api_nodes
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 1s fall 2 rise 1 slowstart 2s
|
||||
server api1 api1:8080 check resolvers docker init-addr none
|
||||
server api2 api2:8080 check resolvers docker init-addr none
|
||||
@@ -0,0 +1,71 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 180
|
||||
spike_limit_mib: 40
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
metric_expiration: 15s
|
||||
add_metric_suffixes: true
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
timeout: 3s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 1
|
||||
queue_size: 64
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 3s
|
||||
max_elapsed_time: 10s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 3s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 1
|
||||
queue_size: 64
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 3s
|
||||
max_elapsed_time: 10s
|
||||
service:
|
||||
telemetry:
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp/tempo]
|
||||
@@ -0,0 +1,805 @@
|
||||
{
|
||||
"uid": "lab6-messenger",
|
||||
"title": "Lab6 | Messenger observability",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": false,
|
||||
"timezone": "browser",
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"tags": [
|
||||
"lab6",
|
||||
"otel"
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "Business API throughput",
|
||||
"description": "Excludes health, counts completed responses.",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s]))",
|
||||
"legendFormat": "all",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(rate(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[30s])) or vector(0)",
|
||||
"legendFormat": "5xx",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Business API error ratio (5xx)",
|
||||
"description": "5xx / all business HTTP responses; threshold 5%.",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "(sum(rate(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s])),0.001)",
|
||||
"legendFormat": "5xx / all",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "API latency p50/p95/p99",
|
||||
"description": "Aggregated histogram buckets across API replicas; seconds, time to response headers.",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])) by (le))",
|
||||
"legendFormat": "p50",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])) by (le))",
|
||||
"legendFormat": "p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])) by (le))",
|
||||
"legendFormat": "p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Jobs per second by outcome",
|
||||
"description": "Attempt outcomes: ready, retry, failed, stale, requeue; not exactly-once audit counts.",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(outcome)(rate(lab_jobs_total[30s]))",
|
||||
"legendFormat": "{{outcome}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Worker processing duration",
|
||||
"description": "",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum(rate(lab_job_duration_seconds_bucket[30s])) by (le))",
|
||||
"legendFormat": "p50",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(lab_job_duration_seconds_bucket[30s])) by (le))",
|
||||
"legendFormat": "p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(lab_job_duration_seconds_bucket[30s])) by (le))",
|
||||
"legendFormat": "p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Time to ready",
|
||||
"description": "Acceptance to durable ready (includes queue, retry and processing).",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum(rate(lab_ready_duration_seconds_bucket[30s])) by (le))",
|
||||
"legendFormat": "p50",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(lab_ready_duration_seconds_bucket[30s])) by (le))",
|
||||
"legendFormat": "p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(lab_ready_duration_seconds_bucket[30s])) by (le))",
|
||||
"legendFormat": "p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Queue depth: durable backlog / RabbitMQ ready",
|
||||
"description": "DB queued includes delayed/unpublished tasks; RabbitMQ ready counts published messages waiting for delivery; neither includes busy jobs.",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(lab_queue_depth)",
|
||||
"legendFormat": "DB queued",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "max(lab_broker_ready)",
|
||||
"legendFormat": "broker ready",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Oldest queued job age",
|
||||
"description": "",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(lab_queue_oldest_age_seconds)",
|
||||
"legendFormat": "age",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "Database pool",
|
||||
"description": "",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "lab_db_pool_used",
|
||||
"legendFormat": "used {{service_instance_id}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "lab_db_pool_waiting",
|
||||
"legendFormat": "waiting {{service_instance_id}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "lab_db_pool_capacity",
|
||||
"legendFormat": "capacity {{service_instance_id}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Worker saturation",
|
||||
"description": "",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(lab_worker_busy) / clamp_min(sum(lab_worker_capacity),1)",
|
||||
"legendFormat": "busy / capacity",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Container memory (or tree RSS fallback)",
|
||||
"description": "",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "lab_container_memory_bytes",
|
||||
"legendFormat": "{{service_name}} / {{service_instance_id}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Container CPU",
|
||||
"description": "CPU cores: 1.0 = one logical core.",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "lab_container_cpu_cores",
|
||||
"legendFormat": "{{service_name}} / {{service_instance_id}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "timeseries",
|
||||
"title": "Collector exporter queue",
|
||||
"description": "",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "otelcol_exporter_queue_size",
|
||||
"legendFormat": "{{exporter}} queue",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "otelcol_exporter_queue_capacity",
|
||||
"legendFormat": "{{exporter}} capacity",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "timeseries",
|
||||
"title": "Collector failed exports",
|
||||
"description": "",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "rate(otelcol_exporter_send_failed_spans[30s]) or rate(otelcol_exporter_send_failed_spans_total[30s])",
|
||||
"legendFormat": "spans {{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "rate(otelcol_exporter_send_failed_log_records[30s]) or rate(otelcol_exporter_send_failed_log_records_total[30s])",
|
||||
"legendFormat": "logs {{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"type": "timeseries",
|
||||
"title": "Alert state",
|
||||
"description": "",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "ALERTS{alertname=~\"Lab.*\"}",
|
||||
"legendFormat": "{{alertname}} / {{alertstate}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"type": "timeseries",
|
||||
"title": "SLI: successful availability (5 min)",
|
||||
"description": "1 \u2212 5xx/all business responses over 5 min; objective >= 99%. No data if no traffic.",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "1 - ((sum(increase(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / sum(increase(lab_http_requests_total{route=~\"/api/.*\"}[5m])))",
|
||||
"legendFormat": "actual SLI",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "vector(0.99)",
|
||||
"legendFormat": "SLO target 99%",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Lab6
|
||||
orgId: 1
|
||||
folder: MTUCI
|
||||
type: file
|
||||
disableDeletion: true
|
||||
allowUiUpdates: false
|
||||
updateIntervalSeconds: 10
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,36 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
uid: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: '-2s'
|
||||
spanEndTimeShift: '2s'
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service_name
|
||||
@@ -0,0 +1,32 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 72h
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,19 @@
|
||||
groups:
|
||||
- name: messenger-lab
|
||||
rules:
|
||||
- alert: LabAPIHighErrorRate
|
||||
expr: (sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])), 0.001) > 0.05
|
||||
for: 9s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Business HTTP 5xx rate exceeds 5% over 30 seconds
|
||||
runbook: REPORT.md#инциденты-и-runbook
|
||||
- alert: LabQueueLag
|
||||
expr: max(lab_queue_oldest_age_seconds) > 15 and max(lab_queue_depth) > 0
|
||||
for: 12s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Oldest queued job exceeds 15 seconds (sustained 12 seconds)
|
||||
runbook: REPORT.md#инциденты-и-runbook
|
||||
@@ -0,0 +1,12 @@
|
||||
global:
|
||||
scrape_interval: 3s
|
||||
evaluation_interval: 3s
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
scrape_configs:
|
||||
- job_name: lab-otlp
|
||||
static_configs:
|
||||
- targets: ['otel-collector:8889']
|
||||
- job_name: collector-self
|
||||
static_configs:
|
||||
- targets: ['otel-collector:8888']
|
||||
@@ -0,0 +1,22 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 72h
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,18 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.2
|
||||
pydantic==2.11.4
|
||||
psycopg[binary]==3.2.6
|
||||
psycopg-pool==3.2.6
|
||||
redis==5.2.1
|
||||
argon2-cffi==23.1.0
|
||||
|
||||
boto3==1.40.0
|
||||
Pillow==11.3.0
|
||||
pika==1.3.2
|
||||
|
||||
opentelemetry-api==1.42.1
|
||||
opentelemetry-sdk==1.42.1
|
||||
opentelemetry-exporter-otlp-proto-http==1.42.1
|
||||
psutil==7.2.2
|
||||
|
||||
PyYAML==6.0.2
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Additional static checks (not the teacher's make check and not a live deploy)."""
|
||||
import ast,json,re
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
|
||||
def main():
|
||||
count=0
|
||||
for path in (ROOT/'app').glob('*.py'):
|
||||
tree=ast.parse(path.read_text(encoding='utf-8'))
|
||||
for n in ast.walk(tree):
|
||||
if isinstance(n,ast.Call) and isinstance(n.func,ast.Attribute) and n.func.attr=='execute' and len(n.args)>1 and isinstance(n.args[0],ast.Constant) and isinstance(n.args[0].value,str) and isinstance(n.args[1],(ast.Tuple,ast.List)):
|
||||
assert n.args[0].value.count('%s')==len(n.args[1].elts),f'SQL binding mismatch: {path.name}:{n.lineno}'
|
||||
count+=1
|
||||
compose=yaml.safe_load((ROOT/'compose.yaml').read_text())
|
||||
for name,svc in compose['services'].items():
|
||||
if 'image' in svc:assert ':' in svc['image'] and not svc['image'].endswith(':latest'),name
|
||||
for port in svc.get('ports',[]):assert port.startswith('127.0.0.1:'),name
|
||||
for mount in svc.get('volumes',[]):
|
||||
src=mount.split(':')[0]
|
||||
assert (ROOT/src).exists() if src.startswith('./') else src in compose['volumes'],f'missing mount {src}'
|
||||
for network in svc.get('networks',[]):assert network in compose['networks']
|
||||
for dep in svc.get('depends_on',{}):assert dep in compose['services']
|
||||
for p in (ROOT/'infra/otel').rglob('*'):
|
||||
if p.suffix in ('.yml','.yaml'):yaml.safe_load(p.read_text())
|
||||
services=compose['services']; assert all('telemetry' in services[s]['networks'] for s in ('api1','api2','dispatcher','worker','otel-collector','prometheus','loki','tempo','grafana'))
|
||||
for v in ('prometheus_data','loki_data','tempo_data','grafana_data'):assert v in compose['volumes']
|
||||
collector=yaml.safe_load((ROOT/'infra/otel/collector.yaml').read_text())
|
||||
for signal in ('metrics','logs','traces'):
|
||||
pipe=collector['service']['pipelines'][signal]
|
||||
assert pipe['receivers']==['otlp'] and pipe['processors']==['memory_limiter','batch']
|
||||
assert all(e in collector['exporters'] for e in pipe['exporters'])
|
||||
for exp in ('otlp/tempo','otlphttp/loki'):
|
||||
conf=collector['exporters'][exp];assert conf['sending_queue']['queue_size']==64 and conf['retry_on_failure']['max_elapsed_time']=='10s'
|
||||
dashboard=json.loads((ROOT/'infra/otel/grafana/dashboards/messenger.json').read_text())
|
||||
expressions='\n'.join(t['expr'] for p in dashboard['panels'] for t in p['targets'])
|
||||
for metric in ('lab_http_requests_total','lab_http_duration_seconds_bucket','lab_jobs_total','lab_job_duration_seconds_bucket','lab_ready_duration_seconds_bucket','lab_queue_depth','lab_queue_oldest_age_seconds','lab_broker_ready','lab_db_pool_used','lab_db_pool_waiting','lab_worker_busy','lab_worker_capacity','lab_container_cpu_cores','lab_container_memory_bytes'):
|
||||
assert metric in expressions,metric
|
||||
assert all(f'histogram_quantile({q}' in expressions for q in (.5,.95,.99))
|
||||
rules=yaml.safe_load((ROOT/'infra/otel/prometheus/alerts.yml').read_text())
|
||||
assert {x['alert'] for g in rules['groups'] for x in g['rules']}=={'LabAPIHighErrorRate','LabQueueLag'}
|
||||
assert all(x.get('for') and x['annotations'].get('runbook') for g in rules['groups'] for x in g['rules'])
|
||||
print(f'PASS: {count} literal SQL bindings; pinned Compose images, mounts and networks; 3 OTLP pipelines, dashboards and alert rules (static only)')
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,11 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
docker compose --env-file .env stop --timeout 5 lb api1 api2 worker dispatcher broker s3 redis postgres grafana otel-collector prometheus loki tempo
|
||||
if errorlevel 1 exit /b 1
|
||||
docker compose --env-file .env up -d --wait postgres redis broker s3 s3-bootstrap prometheus loki tempo otel-collector grafana
|
||||
if errorlevel 1 exit /b 1
|
||||
docker compose --env-file .env up -d --no-deps api1 api2 lb dispatcher worker
|
||||
if errorlevel 1 exit /b 1
|
||||
python scripts\wait_ready.py
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Preserve named volumes. Restart all state stores and both API processes.
|
||||
docker compose --env-file .env stop --timeout 5 lb api1 api2 worker dispatcher broker s3 redis postgres grafana otel-collector prometheus loki tempo
|
||||
docker compose --env-file .env up -d --wait postgres redis broker s3 s3-bootstrap prometheus loki tempo otel-collector grafana
|
||||
docker compose --env-file .env up -d --no-deps api1 api2 lb dispatcher worker
|
||||
python - <<'PY'
|
||||
import time, urllib.request
|
||||
for i in range(120):
|
||||
try:
|
||||
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2) as response:
|
||||
if response.status == 200:
|
||||
print('[restart] all services ready')
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(.5)
|
||||
else:
|
||||
raise SystemExit('[restart] health timeout')
|
||||
PY
|
||||
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if "%TARGET_INSTANCE_ID%"=="api1" goto stop
|
||||
if "%TARGET_INSTANCE_ID%"=="api2" goto stop
|
||||
echo Unexpected TARGET_INSTANCE_ID 1>&2
|
||||
exit /b 2
|
||||
:stop
|
||||
docker compose --env-file .env stop --timeout 1 %TARGET_INSTANCE_ID%
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
# The public failover test sets TARGET_INSTANCE_ID from the actual response header.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
case "${TARGET_INSTANCE_ID:?missing TARGET_INSTANCE_ID}" in
|
||||
api1|api2) docker compose --env-file .env stop --timeout 1 "$TARGET_INSTANCE_ID" ;;
|
||||
*) echo 'Unexpected instance alias' >&2; exit 2 ;;
|
||||
esac
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Operator-facing readiness wait after container replacement."""
|
||||
import time
|
||||
import urllib.request
|
||||
for _ in range(120):
|
||||
try:
|
||||
with urllib.request.urlopen("http://localhost:8080/health/ready", timeout=2) as response:
|
||||
if response.status == 200:
|
||||
print("[restart] all services ready")
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(.5)
|
||||
else:
|
||||
raise SystemExit("[restart] health timeout")
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Concurrent Lab5 uploads (HTTP 202) and eventual processing completion.
|
||||
|
||||
Admission metrics measure upload/enqueue requests, not worker throughput.
|
||||
Completion is checked separately so an accepted-but-stuck job cannot pass.
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from client import Client, check, run
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--concurrency', type=int, default=10)
|
||||
parser.add_argument('--output', default='evidence/attachments-concurrent.json')
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--wait-seconds', type=int, default=180)
|
||||
args = parser.parse_args()
|
||||
check(args.concurrency > 0, 'concurrency must be positive')
|
||||
check(args.wait_seconds > 0, 'wait-seconds must be positive')
|
||||
|
||||
client = Client(args.base_url)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
source = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=parallel.png'
|
||||
|
||||
def upload(_):
|
||||
cli = Client(args.base_url)
|
||||
response = cli.call('POST', path, 202, actor=alice, raw=source,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
att = response['json']
|
||||
check(att['status'] == 'queued', '202 must return a queued attachment')
|
||||
check(att['size_bytes'] == len(source), 'size mismatch')
|
||||
check(att['chat_id'] == chat['id'] and att['owner_id'] == alice['id'],
|
||||
'incorrect attachment ownership')
|
||||
return att['id'], response['elapsed']
|
||||
|
||||
started = time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
results = list(pool.map(upload, range(args.concurrency)))
|
||||
accepted_seconds = time.monotonic() - started
|
||||
ids = [att_id for att_id, _ in results]
|
||||
timings = [duration for _, duration in results]
|
||||
check(len(set(ids)) == args.concurrency, 'duplicate attachment IDs')
|
||||
|
||||
# A 202 response alone only proves acceptance, not successful processing.
|
||||
pending = set(ids)
|
||||
deadline = time.monotonic() + args.wait_seconds
|
||||
while pending and time.monotonic() < deadline:
|
||||
for att_id in list(pending):
|
||||
att = client.call('GET', '/api/v1/attachments/' + att_id,
|
||||
actor=alice, schema='Attachment')['json']
|
||||
check(att['status'] != 'failed', 'attachment reached failed: ' + att_id)
|
||||
if att['status'] == 'ready':
|
||||
variants = [v['name'] for v in att['variants']]
|
||||
check(len(variants) == 3 and set(variants) == {'thumbnail', 'crop', 'optimized'},
|
||||
'missing/duplicate image variants: ' + att_id)
|
||||
pending.remove(att_id)
|
||||
if pending:
|
||||
time.sleep(0.5)
|
||||
check(not pending, 'attachments not ready before deadline: ' + str(len(pending)))
|
||||
complete_seconds = time.monotonic() - started
|
||||
|
||||
data = {
|
||||
'concurrency': args.concurrency,
|
||||
'accepted_uploads': len(results),
|
||||
'ready_attachments': args.concurrency,
|
||||
'failed_requests': 0,
|
||||
'acceptance_elapsed_seconds': round(accepted_seconds, 3),
|
||||
'acceptance_rps': round(len(results) / accepted_seconds, 3),
|
||||
'upload_latency_p50_ms': round(statistics.median(timings) * 1000, 1),
|
||||
'upload_latency_max_ms': round(max(timings) * 1000, 1),
|
||||
'all_ready_elapsed_seconds': round(complete_seconds, 3),
|
||||
'bytes_per_file': len(source),
|
||||
'note': 'HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats.',
|
||||
}
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(exist_ok=True, parents=True)
|
||||
output.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('PASS: %d concurrent HTTP 202 uploads -> ready, 3 variants each' % args.concurrency)
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab5 negative API tests with isolated synthetic users (requires live stack)."""
|
||||
from urllib.parse import quote
|
||||
import uuid
|
||||
from client import Client, arguments, run, check
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Lab5 negative file/ACL/security tests')
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
base = '/api/v1/chats/' + chat['id'] + '/attachments?filename='
|
||||
data = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
|
||||
c.call('POST', base+'big.png', 413, actor=alice, raw=data+b'\x00'*(10*1024*1024+1-len(data)), headers={'Content-Type':'image/png'})
|
||||
c.call('POST', base+'bad.png', 415, actor=alice, raw=b'not-an-image', headers={'Content-Type':'image/png'})
|
||||
c.call('POST', base+'fake.jpg', 415, actor=alice, raw=data, headers={'Content-Type':'image/jpeg'})
|
||||
c.call('POST', base+quote('../escape.png',safe=''), 400, actor=alice, raw=data, headers={'Content-Type':'image/png'})
|
||||
c.call('POST', base+'x.png', 404, actor=eve, raw=data, headers={'Content-Type':'image/png'})
|
||||
c.call('POST', base+'x.png', 401, raw=data, headers={'Content-Type':'image/png'})
|
||||
uploaded = c.attachment(alice,chat)[0]
|
||||
ready = c.wait_ready(alice, uploaded)
|
||||
unknown = str(uuid.uuid4())
|
||||
c.call('GET','/api/v1/attachments/'+unknown,404,actor=alice)
|
||||
c.call('GET','/api/v1/attachments/'+uploaded['id'],404,actor=eve)
|
||||
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',404,actor=eve)
|
||||
# A ready Lab5 attachment has a downloadable thumbnail for chat members.
|
||||
# A non-member still gets 404, and an unknown variant must not resolve.
|
||||
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',404,actor=eve)
|
||||
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=nonexistent',404,actor=alice)
|
||||
check(any(v['name'] == 'thumbnail' for v in ready['variants']), 'missing thumbnail variant')
|
||||
thumb = c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',200,actor=alice)
|
||||
check(thumb['headers'].get_content_type() == 'image/jpeg', 'wrong thumbnail MIME')
|
||||
other_chat = c.call('POST','/api/v1/chats',201,actor=alice,
|
||||
body={'title':'Другой чат','member_ids':[alice['id'],bob['id']]},schema='Chat')['json']
|
||||
path = '/api/v1/chats/'+other_chat['id']+'/messages'
|
||||
c.call('POST',path,404,actor=alice,
|
||||
body={'text':'cross-chat','attachment_ids':[uploaded['id']]},
|
||||
headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',404,actor=alice,
|
||||
body={'text':'unknown','attachment_ids':[unknown]},
|
||||
headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
ok = c.message(bob,chat,'',attachments=[uploaded['id']])['json']
|
||||
check(ok['attachment_ids']==[uploaded['id']], 'valid attachment not linked')
|
||||
print('PASS: lab5 negative image, MIME, size, filename, ACL, variants and cross-chat checks')
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab3 own test: 30 requests using identical actor/chat/key/body across both replicas."""
|
||||
import concurrent.futures
|
||||
import sys
|
||||
import uuid
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Check atomic idempotency across API replicas")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
body = {"text": "Concurrent replay " + key}
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
def worker(_):
|
||||
# Independent request client; one shared bearer token and idempotency key.
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
return client.call("POST", path, expected=201, actor=alice, body=body,
|
||||
headers={"Idempotency-Key": key}, schema="Message")
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor:
|
||||
results = list(executor.map(worker, range(30)))
|
||||
ids = {result["json"]["id"] for result in results}
|
||||
instances = {result["headers"]["X-Instance-Id"] for result in results}
|
||||
check(len(ids) == 1, "parallel replay created different messages")
|
||||
check(len(instances) >= 2, "LB did not serve requests via both API replicas")
|
||||
check(all(r["json"] == results[0]["json"] for r in results), "replay returned altered result")
|
||||
c.call("POST", path, 409, actor=alice, body={"text": "different body"}, headers={"Idempotency-Key": key})
|
||||
c.call("POST", path, 201, actor=bob, body=body, headers={"Idempotency-Key": key}, schema="Message")
|
||||
history = c.call("GET", path + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"]
|
||||
check(sum(x["id"] in ids for x in history) == 1, "message duplicated in history")
|
||||
print("PASS: parallel idempotency across", len(instances), "replicas; one message, replay 201, conflicting body 409")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prevent a mismatch between SQL placeholders and parameter tuple in worker result persistence."""
|
||||
import ast
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
source = (Path(__file__).resolve().parents[1] / "app" / "jobs.py").read_text(encoding="utf-8")
|
||||
tree = ast.parse(source)
|
||||
fn = next(x for x in tree.body if isinstance(x, ast.FunctionDef) and x.name == "publish_result")
|
||||
matches = []
|
||||
for node in ast.walk(fn):
|
||||
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
|
||||
continue
|
||||
if node.func.attr != "execute" or len(node.args) < 2:
|
||||
continue
|
||||
if not isinstance(node.args[0], ast.Constant) or not isinstance(node.args[0].value, str):
|
||||
continue
|
||||
sql = node.args[0].value
|
||||
if "INSERT INTO attachment_variants" not in sql:
|
||||
continue
|
||||
assert isinstance(node.args[1], ast.Tuple), "Expected literal parameters tuple"
|
||||
placeholders = sql.count("%s")
|
||||
params = len(node.args[1].elts)
|
||||
columns_match = re.search(r"INSERT INTO attachment_variants\s*\(([^)]*)\)", sql, re.I | re.S)
|
||||
assert columns_match, "Columns missing in INSERT"
|
||||
columns = len(columns_match.group(1).split(","))
|
||||
assert placeholders == columns == params, f"SQL mismatch: {columns} columns, {placeholders} placeholders, {params} parameters"
|
||||
matches.append((columns, placeholders, params))
|
||||
assert len(matches) == 1, "Expected exactly one attachment_variants INSERT"
|
||||
print("PASS: attachment_variants INSERT has 9 columns, 9 placeholders, 9 arguments")
|
||||
@@ -0,0 +1,341 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab6 live evidence. Only synthetic test stack; no volume removal.
|
||||
Run scenarios separately. Queries stored in evidence, no HTTP auth or payloads.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import math
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
from client import Client, Failure, check, run
|
||||
from pipeline_extended import compose, pg, scalar, wait_for
|
||||
from pipeline_resilience import verify_ready
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
PROM='http://127.0.0.1:9090'
|
||||
LOKI='http://127.0.0.1:3100'
|
||||
TEMPO='http://127.0.0.1:3200'
|
||||
GRAFANA='http://127.0.0.1:3000'
|
||||
ERR='(sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])),0.001)'
|
||||
LAG='max(lab_queue_oldest_age_seconds)'
|
||||
DEPTH='max(lab_queue_depth)'
|
||||
FAILED='sum(otelcol_exporter_send_failed_spans) or sum(otelcol_exporter_send_failed_spans_total) or vector(0)'
|
||||
# Explicit Prometheus readers can append unit suffixes to Collector self-metrics.
|
||||
# Missing gauges are not zero: absent telemetry must fail the outage test.
|
||||
QUEUE='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_size(_batches|_requests|_items)?"})'
|
||||
QUEUE_CAPACITY='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_capacity(_batches|_requests|_items)?"})'
|
||||
RSS='max({job="collector-self",__name__=~"otelcol_process_memory_rss(_bytes)?"})'
|
||||
|
||||
|
||||
def get(url, params=None, *, as_json=True):
|
||||
try:
|
||||
with urllib.request.urlopen(url+('?' + urllib.parse.urlencode(params) if params else ''),timeout=8) as r:
|
||||
raw=r.read(32*1024*1024)
|
||||
return json.loads(raw) if as_json else raw.decode('utf-8',errors='replace')
|
||||
except (urllib.error.URLError,TimeoutError,ValueError) as exc:
|
||||
raise Failure('diagnostic endpoint unavailable: '+url.split('?')[0]+' ('+type(exc).__name__+')') from None
|
||||
|
||||
|
||||
def prom(expr):
|
||||
data=get(PROM+'/api/v1/query',{'query':expr})
|
||||
check(data.get('status')=='success','PromQL query failed')
|
||||
return data['data']['result']
|
||||
|
||||
|
||||
def val(expr):
|
||||
rows=prom(expr)
|
||||
return float(rows[0]['value'][1]) if rows else None
|
||||
|
||||
|
||||
def alert_state(name):
|
||||
rules=get(PROM+'/api/v1/rules')['data']['groups']
|
||||
for group in rules:
|
||||
for rule in group['rules']:
|
||||
if rule['name']==name:
|
||||
check(rule.get('health')=='ok','alert rule is not healthy: '+name)
|
||||
return rule['state']
|
||||
raise Failure('alert rule missing: '+name)
|
||||
|
||||
|
||||
def wait_alert(name,state,seconds=100):
|
||||
return wait_for(name+' '+state,lambda:alert_state(name),lambda x:x==state,seconds=seconds,interval=2)
|
||||
|
||||
|
||||
def ready():
|
||||
for url in (PROM+'/-/ready',LOKI+'/ready',TEMPO+'/ready'):
|
||||
wait_for(url,lambda:get(url,as_json=False),lambda x:bool(x),seconds=100,interval=2)
|
||||
wait_for('Grafana',lambda:get(GRAFANA+'/api/health'),lambda x:x.get('database')=='ok',seconds=100)
|
||||
wait_for('Collector scrape',lambda:val('up{job="collector-self"}'),lambda v:v==1,seconds=40)
|
||||
wait_for('OTLP metrics scrape',lambda:val('up{job="lab-otlp"}'),lambda v:v==1,seconds=40)
|
||||
|
||||
|
||||
def trace_spans(tid):
|
||||
raw=get(TEMPO+'/api/traces/'+tid)
|
||||
spans=[]
|
||||
def walk(obj):
|
||||
if isinstance(obj,dict):
|
||||
if {'spanId','traceId','name'}<=set(obj): spans.append(obj)
|
||||
else:
|
||||
for v in obj.values():walk(v)
|
||||
elif isinstance(obj,list):
|
||||
for v in obj:walk(v)
|
||||
walk(raw)
|
||||
return spans
|
||||
|
||||
|
||||
def logs(tid,service,request_id=None):
|
||||
expr='{service_name="%s"} | trace_id="%s"' % (service,tid)
|
||||
if request_id: expr+=' | request_id="%s"'%request_id
|
||||
data=get(LOKI+'/loki/api/v1/query_range',{'query':expr,'start':str(int((time.time()-900)*1e9)), 'end':str(time.time_ns()),'limit':100})
|
||||
return data['data']['result']
|
||||
|
||||
|
||||
def upload(c,alice,chat):
|
||||
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
r=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
tid=r['headers'].get('X-Trace-Id','')
|
||||
check(len(tid)==32 and int(tid,16)>0,'missing active API trace ID: check SDK enablement')
|
||||
return r['json'],raw,tid,r['headers']['X-Request-Id']
|
||||
|
||||
|
||||
def span_attrs(s):
|
||||
attrs={}
|
||||
for a in s.get('attributes',[]):
|
||||
value=a.get('value',{})
|
||||
attrs[a['key']]=next(iter(value.values()),None)
|
||||
return attrs
|
||||
|
||||
|
||||
def chain_ok(spans):
|
||||
names={s['name'] for s in spans}
|
||||
required={'POST /api/v1/chats/{chat_id}/attachments','rabbitmq.publish','rabbitmq.consume','s3.upload_fileobj','s3.get_object','s3.put_object','db.publish_result'}
|
||||
if not required<=names:return False
|
||||
producers={s['spanId'] for s in spans if s['name']=='rabbitmq.publish'}
|
||||
consumers=[s for s in spans if s['name']=='rabbitmq.consume']
|
||||
outcomes={span_attrs(s).get('job.outcome') for s in consumers}
|
||||
return len(producers)>=2 and {'retry','ready'}<=outcomes and all(s.get('parentSpanId') in producers for s in consumers)
|
||||
|
||||
|
||||
def trace_retry(c):
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
compose('stop','worker')
|
||||
try:
|
||||
att,raw,tid,rid=upload(c,alice,chat)
|
||||
# Override only worker, never a public endpoint; reset in finally.
|
||||
env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'once'}
|
||||
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env=env,check=True,timeout=180)
|
||||
verify_ready(c,bob,att,raw,seconds=180)
|
||||
spans=wait_for('stored retry trace',lambda:trace_spans(tid),chain_ok,seconds=100,interval=3)
|
||||
wait_for('API log joined by request/trace ID',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
|
||||
def worker_events(streams):
|
||||
return {v[1] for row in streams for v in row.get('values',[])}
|
||||
wlogs=wait_for('worker retry and ready logs',lambda:logs(tid,'messenger-worker'),lambda rows:{'job.retry','job.ready'}<=worker_events(rows),seconds=90)
|
||||
# Keep only allowlisted names/IDs, not arbitrary span values or log payloads.
|
||||
summary=[{'name':s['name'],'id':s['spanId'],'parent':s.get('parentSpanId'),'attempt':span_attrs(s).get('job.attempt'),'outcome':span_attrs(s).get('job.outcome')} for s in spans]
|
||||
result={'trace_id':tid,'request_id':rid,'spans':summary,'worker_log_streams':len(wlogs),'worker_events':sorted(worker_events(wlogs)&{'job.claimed','job.retry','job.ready','image.transformed'}),'retry_chain':True}
|
||||
print('PASS trace-retry: API/producer/consumer/S3/DB with retry, correlated logs',tid)
|
||||
return result
|
||||
finally:
|
||||
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'off'},check=True,timeout=180)
|
||||
|
||||
|
||||
def queue_lag(c):
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
wait_alert('LabQueueLag','inactive')
|
||||
compose('stop','worker')
|
||||
try:
|
||||
att,raw,tid,rid=upload(c,alice,chat)
|
||||
t=time.monotonic()
|
||||
wait_alert('LabQueueLag','firing')
|
||||
lag=val(LAG);depth=val(DEPTH)
|
||||
check(lag is not None and lag>15 and depth>=1,'firing not supported by actual backlog metrics')
|
||||
detected=time.monotonic()-t
|
||||
print('Queue alert firing: oldest age',lag,'s, queued',depth,flush=True)
|
||||
compose('up','-d','worker')
|
||||
verify_ready(c,bob,att,raw,seconds=180)
|
||||
wait_alert('LabQueueLag','inactive',seconds=120)
|
||||
wait_for('queue drained',lambda:val(DEPTH),lambda v:v==0,seconds=60,interval=2)
|
||||
print('PASS queue-lag: firing -> ready -> resolved')
|
||||
return {'trace_id':tid,'firing_lag_seconds':lag,'firing_depth':depth,'detected_seconds':round(detected,3),'resolved_depth':val(DEPTH)}
|
||||
finally:compose('up','-d','worker')
|
||||
|
||||
|
||||
def db_error(c):
|
||||
alice,_,_,_=c.bootstrap()
|
||||
wait_alert('LabAPIHighErrorRate','inactive')
|
||||
user=compose('exec','-T','postgres','printenv','POSTGRES_USER')
|
||||
db=compose('exec','-T','postgres','printenv','POSTGRES_DB')
|
||||
# Bounded, isolated table lock. Health SELECT 1 works, authenticated business
|
||||
# read times out at statement_timeout. Never stop DB (LB would hide API 5xx).
|
||||
cmd=['docker','compose','--env-file','.env','exec','-T','postgres','psql','-X','-q','-v','ON_ERROR_STOP=1','-U',user,'-d',db,'-c','BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(60); COMMIT;']
|
||||
lock=subprocess.Popen(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
wait_for('users lock held',lambda:scalar("SELECT count(*) FROM pg_locks WHERE relation='users'::regclass AND mode='AccessExclusiveLock' AND granted"),lambda v:int(v)>0,seconds=15)
|
||||
start=time.monotonic(); traces=[]; codes=[]
|
||||
while time.monotonic()-start < 43:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
|
||||
def get_user(_):
|
||||
r=Client(timeout=12).call('GET','/api/v1/users/me',[200,503],actor=alice,schema='User')
|
||||
return r['status'],r['headers'].get('X-Trace-Id'),r['headers'].get('X-Request-Id')
|
||||
for code,tid,rid in pool.map(get_user,range(3)):
|
||||
codes.append(code)
|
||||
if code==503 and tid:traces.append((tid,rid))
|
||||
if traces and alert_state('LabAPIHighErrorRate')=='firing':break
|
||||
check(bool(traces),'no real HTTP 503 from DB lock')
|
||||
wait_alert('LabAPIHighErrorRate','firing',seconds=20)
|
||||
ratio=val(ERR)
|
||||
check(ratio is not None and ratio>0.05,'error ratio below threshold')
|
||||
tid,rid=traces[0]
|
||||
spans=wait_for('DB timeout trace',lambda:trace_spans(tid),lambda v:any(s['name']=='db.transaction' and span_attrs(s).get('error.type')=='QueryCanceled' for s in v),seconds=60,interval=2)
|
||||
wait_for('error log',lambda:logs(tid,'messenger-api',rid),bool,seconds=60)
|
||||
print('Error alert firing, database QueryCanceled visible; ratio',ratio,flush=True)
|
||||
finally:
|
||||
# SQL transaction has server-side 60s bound; wait for normal COMMIT.
|
||||
try:lock.wait(timeout=75)
|
||||
except subprocess.TimeoutExpired:
|
||||
lock.terminate();lock.wait(timeout=10)
|
||||
# Keep successful business traffic in the evaluation window to prove recovery.
|
||||
t=time.monotonic()
|
||||
while time.monotonic()-t<65:
|
||||
c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
|
||||
if alert_state('LabAPIHighErrorRate')=='inactive' and val(ERR) is not None and val(ERR)<=.05:break
|
||||
time.sleep(1)
|
||||
wait_alert('LabAPIHighErrorRate','inactive',seconds=40)
|
||||
print('PASS db-error: lock -> QueryCanceled/503 -> firing -> resolved')
|
||||
return {'trace_id':tid,'request_id':rid,'http_codes':codes,'firing_error_ratio':ratio,'resolved_error_ratio':val(ERR),'error_spans':[{'name':s['name'],'error_type':span_attrs(s).get('error.type')} for s in spans if span_attrs(s).get('error.type')]}
|
||||
|
||||
|
||||
def collector_sample():
|
||||
check(val('up{job="collector-self"}')==1,'Collector self-metrics scrape down')
|
||||
rss,queue,capacity=val(RSS),val(QUEUE),val(QUEUE_CAPACITY)
|
||||
check(rss is not None and math.isfinite(rss) and rss>0,
|
||||
'Collector RSS metric absent or invalid; query otelcol_process_memory_rss(_bytes)? in Prometheus')
|
||||
check(queue is not None and math.isfinite(queue) and queue>=0,
|
||||
'Tempo exporter queue size metric absent or invalid; inspect collector-self metrics')
|
||||
check(capacity is not None and math.isfinite(capacity) and capacity==64,
|
||||
'Tempo exporter queue capacity absent or differs from configured 64: '+str(capacity))
|
||||
return rss,queue,capacity
|
||||
|
||||
|
||||
def backend_down(c):
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
baseline=val(FAILED) or 0
|
||||
cid=compose('ps','-q','otel-collector')
|
||||
before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
|
||||
memory_limit=int(subprocess.check_output(['docker','inspect','--format','{{.HostConfig.Memory}}',cid],text=True).strip())
|
||||
check(memory_limit==256*1024*1024,'unexpected Collector container memory limit: '+str(memory_limit))
|
||||
rss,queue,capacity=collector_sample()
|
||||
print(f'Collector baseline: RSS={rss/1024**2:.2f} MiB, queue={queue:g}/{capacity:g}, limit={memory_limit/1024**2:g} MiB',flush=True)
|
||||
compose('stop','tempo')
|
||||
try:
|
||||
start=time.monotonic();n=0;memory=[];queues=[];failures=[]
|
||||
att,raw,tid,_=upload(c,alice,chat)
|
||||
while time.monotonic()-start<40:
|
||||
c.message(alice,chat,'telemetry-backend-offline')
|
||||
n+=1
|
||||
rss,queue,capacity=collector_sample()
|
||||
memory.append(rss);queues.append(queue)
|
||||
x=val(FAILED)
|
||||
if x is not None:failures.append(x)
|
||||
time.sleep(.3)
|
||||
verify_ready(c,bob,att,raw,seconds=180)
|
||||
check(n>0,'no successful writes during backend outage')
|
||||
check(failures and max(failures)>baseline,'no recorded exporter failure; inspect Collector internal metrics')
|
||||
rss,queue,capacity=collector_sample()
|
||||
memory.append(rss);queues.append(queue)
|
||||
print(f'Collector outage: sampled RSS max={max(memory)/1024**2:.2f} MiB, queue max={max(queues):g}/{capacity:g}, successful writes={n}',flush=True)
|
||||
after=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
|
||||
check(after==before and after.endswith('false'),'Collector restarted or OOM-killed: '+before+' -> '+after)
|
||||
check(max(memory)<memory_limit,f'Collector RSS {max(memory):g} bytes exceeds/equal to container limit {memory_limit} bytes')
|
||||
check(max(queues)<=capacity,'collector queue exceeds configured cap')
|
||||
result={'successful_messages_during_tempo_outage':n,'trace_id':tid,'max_sampled_collector_rss_bytes':max(memory),'collector_memory_limit_bytes':memory_limit,'rss_sample_count':len(memory),'max_export_queue':max(queues),'export_queue_capacity':capacity,'failed_span_counter_delta':max(failures)-baseline,'restart_state':after}
|
||||
finally:compose('up','-d','tempo')
|
||||
ready()
|
||||
# Confirm a fresh post-recovery trace can be retrieved.
|
||||
r=c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
|
||||
tid2=r['headers']['X-Trace-Id']
|
||||
wait_for('post-recovery trace',lambda:trace_spans(tid2),bool,seconds=90)
|
||||
result['recovered_trace_id']=tid2
|
||||
print('PASS backend-down: writes and image processing persisted, telemetry failed visibly; queue/RAM bounded, fresh trace recovered')
|
||||
return result
|
||||
|
||||
|
||||
def metrics_snapshot(c):
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
att,raw,_,_=upload(c,alice,chat)
|
||||
verify_ready(c,bob,att,raw,seconds=150)
|
||||
until=time.monotonic()+12
|
||||
while time.monotonic()<until:
|
||||
c.message(alice,chat,'synthetic-metric-check')
|
||||
time.sleep(5)
|
||||
# Read the real provisioned Grafana dashboard/datasource list, without
|
||||
# printing credentials or copying the response into the evidence.
|
||||
secret=compose('exec','-T','grafana','printenv','GF_SECURITY_ADMIN_PASSWORD')
|
||||
auth=base64.b64encode(('admin:'+secret).encode()).decode()
|
||||
def grafana_json(path):
|
||||
r=urllib.request.Request(GRAFANA+path,headers={'Authorization':'Basic '+auth})
|
||||
try:
|
||||
with urllib.request.urlopen(r,timeout=8) as f:return json.load(f)
|
||||
except (urllib.error.URLError,TimeoutError,ValueError):
|
||||
raise Failure('Grafana authenticated provisioning check failed; verify admin password/volume') from None
|
||||
dashboard=grafana_json('/api/dashboards/uid/lab6-messenger')['dashboard']
|
||||
check(len(dashboard['panels'])>=15,'provisioned dashboard incomplete')
|
||||
check({'prometheus','loki','tempo'}<={d['uid'] for d in grafana_json('/api/datasources')},'provisioned datasources missing')
|
||||
queries={'api_rps':'sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))','error_ratio':ERR,
|
||||
'p50':'histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
|
||||
'p95':'histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
|
||||
'p99':'histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
|
||||
'jobs':'lab_jobs_total','job_histogram':'lab_job_duration_seconds_count','ready_histogram':'lab_ready_duration_seconds_count',
|
||||
'depth':DEPTH,'oldest':LAG,'broker_ready':'lab_broker_ready','broker_consumers':'lab_broker_consumers','pool_used':'lab_db_pool_used','pool_waiting':'lab_db_pool_waiting',
|
||||
'capacity':'lab_worker_capacity','busy':'lab_worker_busy','cpu':'lab_container_cpu_cores','memory':'lab_container_memory_bytes'}
|
||||
output={}
|
||||
for key,expr in queries.items():
|
||||
rows=wait_for('metric '+key,lambda:prom(expr),bool,seconds=45,interval=2)
|
||||
if key in ('p50','p95','p99'): check(math.isfinite(float(rows[0]['value'][1])),'percentile has no finite datapoints; repeat after active load')
|
||||
output[key]={'query':expr,'result':rows}
|
||||
forbidden={'user_id','request_id','chat_id','message_id','attachment_id','filename','job_id','trace_id','span_id'}
|
||||
for item in output.values():
|
||||
check(all(not forbidden.intersection(r.get('metric',{})) for r in item['result']),'high-cardinality metric labels')
|
||||
print('PASS metrics snapshot: all expected series present, no forbidden ID labels')
|
||||
return output
|
||||
|
||||
|
||||
def history(c):
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
att,raw,tid,rid=upload(c,alice,chat)
|
||||
verify_ready(c,bob,att,raw,seconds=150)
|
||||
wait_for('stored trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=90)
|
||||
wait_for('stored API log',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
|
||||
time.sleep(6)
|
||||
historic_time=time.time()
|
||||
expr='sum(lab_http_requests_total{route=~"/api/.*"})'
|
||||
def historic():
|
||||
rows=get(PROM+'/api/v1/query',{'query':expr,'time':str(historic_time)})['data']['result']
|
||||
return float(rows[0]['value'][1]) if rows else None
|
||||
before=historic();check(before is not None and before>0,'no historic metric')
|
||||
compose('restart','--timeout','20','prometheus','loki','tempo',timeout=180)
|
||||
ready()
|
||||
wait_for('historical trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=100)
|
||||
wait_for('historical log',lambda:logs(tid,'messenger-api',rid),bool,seconds=100)
|
||||
after=historic();check(after==before,'historic metric changed or disappeared')
|
||||
print('PASS history: saved metrics, logs and traces survive backend restart')
|
||||
return {'trace_id':tid,'historical_query_timestamp':historic_time,'metric_before':before,'metric_after':after}
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--scenario',choices=['trace-retry','queue-lag','db-error','backend-down','metrics','history'],required=True);args=p.parse_args()
|
||||
check(os.getenv('LAB6_ALLOW_FAILURE_TEST')=='yes','Set LAB6_ALLOW_FAILURE_TEST=yes on isolated synthetic stack')
|
||||
ready();c=Client(timeout=12)
|
||||
fun={'trace-retry':trace_retry,'queue-lag':queue_lag,'db-error':db_error,'backend-down':backend_down,'metrics':metrics_snapshot,'history':history}[args.scenario]
|
||||
result=fun(c)
|
||||
file=ROOT/'evidence'/('lab6-'+args.scenario+'.json');file.write_text(json.dumps({'scenario':args.scenario,'status':'PASS','result':result},indent=2,ensure_ascii=False)+'\n',encoding='utf-8')
|
||||
print('Saved',file.relative_to(ROOT))
|
||||
|
||||
if __name__=='__main__':run(main)
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fast non-Docker checks for input contracts and cursor scoping."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from app.core import make_cursor, read_cursor, parse_limit, APIError
|
||||
from app.models import CreateUser, CreateChat, CreateMessage
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
||||
def reject(fn):
|
||||
try:
|
||||
fn()
|
||||
except (APIError, ValidationError, ValueError):
|
||||
return
|
||||
raise AssertionError("Expected invalid input")
|
||||
|
||||
|
||||
def main():
|
||||
token = str(uuid.uuid4())
|
||||
dt = datetime.now(timezone.utc)
|
||||
cursor = make_cursor("messages", token, dt, str(uuid.uuid4()))
|
||||
assert read_cursor(cursor, "messages", token)[0] == dt
|
||||
reject(lambda: read_cursor(cursor, "messages", str(uuid.uuid4())))
|
||||
reject(lambda: read_cursor(cursor, "chats", token))
|
||||
assert parse_limit(None) == 50
|
||||
reject(lambda: parse_limit("0"))
|
||||
reject(lambda: parse_limit("101"))
|
||||
reject(lambda: CreateUser(username="user_name", display_name=" \t", password="Long-enough-secret"))
|
||||
reject(lambda: CreateUser(username="user_name", display_name="Test", password=" " * 3))
|
||||
assert CreateMessage(text="A\u0000B").text == "A\u0000B"
|
||||
reject(lambda: CreateMessage(text=" \t"))
|
||||
reject(lambda: CreateChat(title="title", member_ids=[token] * 2))
|
||||
reject(lambda: CreateChat(title="title", member_ids=[token] * 101))
|
||||
print("PASS: models, cursor scoping, limits, Unicode U+0000")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Additional *live* Lab5 fault tests; operator-controlled, synthetic records only.
|
||||
|
||||
broker-persistence: require a publisher-confirmed RabbitMQ message to survive
|
||||
broker restart without a running dispatcher or worker. This differs from the
|
||||
outbox-restoration tests where dispatcher publishes *after* broker recovery.
|
||||
|
||||
failed-replay: safely inject a deterministic checksum mismatch into one fresh
|
||||
synthetic attachment's DB metadata, verify permanent failure, restore the
|
||||
original hash, replay via the operator-only CLI, and verify success plus audit.
|
||||
|
||||
staging-cleanup: verify the staged S3 object is absent after ready (with a
|
||||
matching cleared staging_key in the DB).
|
||||
|
||||
No volumes are deleted. The test may stop/restart worker/dispatcher/broker;
|
||||
run only against a disposable LAB5 environment and never concurrently.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from client import Client, Failure, check, run
|
||||
from pipeline_resilience import verify_ready, wait_api
|
||||
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, expected=(0,), timeout=240):
|
||||
proc = subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
|
||||
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout,
|
||||
)
|
||||
if proc.returncode not in expected:
|
||||
# Don't print env vars, tokens or command stdout/stderr by default.
|
||||
raise Failure('docker compose %s failed (exit %d); inspect ps/logs' %
|
||||
(' '.join(args[:2]), proc.returncode))
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def pg(sql):
|
||||
"""Read-only or deliberately scoped fault SQL; UUIDs never user-supplied here."""
|
||||
user = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_USER').strip()
|
||||
dbname = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_DB').strip()
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-X', '-q', '-A', '-t',
|
||||
'-v', 'ON_ERROR_STOP=1', '-U', user, '-d', dbname, '-c', sql)
|
||||
|
||||
|
||||
def scalar(sql):
|
||||
return pg(sql).splitlines()[-1].strip()
|
||||
|
||||
|
||||
def wait_for(label, action, predicate, seconds=30, interval=0.5):
|
||||
deadline = time.monotonic() + seconds
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
last = action()
|
||||
if predicate(last):
|
||||
return last
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(interval)
|
||||
raise Failure('timed out waiting for ' + label + ', last=' + str(last)[:100])
|
||||
|
||||
|
||||
def job_query(att_id, fields):
|
||||
uuid.UUID(att_id)
|
||||
return scalar('SELECT %s FROM processing_jobs WHERE attachment_id = \'%s\'' %
|
||||
(fields, att_id))
|
||||
|
||||
|
||||
def queue_ready():
|
||||
out = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', 'list_queues',
|
||||
'name', 'messages_ready', 'messages_unacknowledged')
|
||||
for line in out.splitlines():
|
||||
parts = re.split(r'\s+', line.strip())
|
||||
if parts[0] == 'attachment-processing.v1' and len(parts) >= 3:
|
||||
return int(parts[1]), int(parts[2])
|
||||
raise Failure('RabbitMQ durable queue missing')
|
||||
|
||||
|
||||
def make_upload(c):
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
att, original = c.attachment(alice, chat)
|
||||
check(att['status'] == 'queued', 'upload must return 202 queued')
|
||||
uuid.UUID(att['id'])
|
||||
return alice, bob, att, original
|
||||
|
||||
|
||||
def wait_staging_cleared(att_id, seconds=20):
|
||||
return wait_for('staging_key cleared after ready',
|
||||
lambda: scalar("SELECT COALESCE(staging_key,'<NULL>') FROM attachments WHERE id='%s'" % att_id),
|
||||
lambda value: value == '<NULL>', seconds=seconds)
|
||||
|
||||
|
||||
def verify_staging_absent(att_id):
|
||||
# This executes *inside* the worker container with its normal S3 credentials.
|
||||
# Never expose S3 secrets to the host or write them to evidence files.
|
||||
code = '''import sys
|
||||
from botocore.exceptions import ClientError
|
||||
from app.attachments import s3_client, BUCKET
|
||||
key='staging/'+sys.argv[1]+'/original'
|
||||
try:
|
||||
s3_client().head_object(Bucket=BUCKET,Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get('ResponseMetadata',{}).get('HTTPStatusCode')==404:
|
||||
print('ABSENT')
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
print('STILL_PRESENT')'''
|
||||
result = compose('exec', '-T', 'worker', 'python', '-c', code, att_id)
|
||||
check(result == 'ABSENT', 'staging S3 object still present: ' + result[:70])
|
||||
|
||||
|
||||
def broker_persistence(c):
|
||||
compose('stop', 'worker')
|
||||
compose('up', '-d', 'dispatcher')
|
||||
baseline = queue_ready()[0]
|
||||
alice, _bob, att, original = make_upload(c)
|
||||
print('202 queued:', att['id'], flush=True)
|
||||
sql = "SELECT COALESCE(bool_and(published_at IS NOT NULL),false) FROM job_outbox WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % att['id']
|
||||
wait_for('outbox publisher confirm', lambda: scalar(sql), lambda v: v == 't', seconds=30)
|
||||
wait_for('message in RabbitMQ ready queue', lambda: queue_ready()[0],
|
||||
lambda n: n >= baseline + 1, seconds=15)
|
||||
compose('stop', 'dispatcher')
|
||||
before = queue_ready()[0]
|
||||
check(before >= baseline + 1, 'published message disappeared before restart')
|
||||
print('Publisher confirmed; dispatcher and worker stopped; broker queue ready=%d' % before, flush=True)
|
||||
compose('restart', 'broker')
|
||||
compose('up', '-d', '--wait', 'broker')
|
||||
after = wait_for('RabbitMQ queue restoration', lambda: queue_ready()[0],
|
||||
lambda n: n >= before, seconds=40, interval=1)
|
||||
check(scalar(sql) == 't', 'outbox publish confirmation lost')
|
||||
print('Broker restarted; ready messages persisted=%d' % after, flush=True)
|
||||
# CRITICAL: do NOT restart dispatcher yet; only the restored RabbitMQ message
|
||||
# can deliver this job to the worker.
|
||||
compose('up', '-d', 'worker')
|
||||
verify_ready(c, alice, att, original, seconds=180)
|
||||
check(job_query(att['id'], 'status') == 'ready', 'job not committed ready')
|
||||
print('PASS: broker-persistence, confirmed RabbitMQ message survives broker restart without outbox republish')
|
||||
|
||||
|
||||
def failed_replay(c):
|
||||
compose('stop', 'worker', 'dispatcher')
|
||||
alice, _bob, att, original = make_upload(c)
|
||||
ident = att['id']
|
||||
original_sha = hashlib.sha256(original).hexdigest()
|
||||
check(original_sha == att['sha256'], 'fixture hash mismatch')
|
||||
check(scalar("SELECT sha256 FROM attachments WHERE id='%s'" % ident) == original_sha,
|
||||
'DB hash unexpected before controlled fault')
|
||||
fault_sha = '0' * 64 if original_sha != '0' * 64 else 'f' * 64
|
||||
changed = False
|
||||
try:
|
||||
updated = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='queued' RETURNING id" %
|
||||
(fault_sha, ident))
|
||||
check(ident in updated, 'failed to inject synthetic checksum mismatch')
|
||||
changed = True
|
||||
compose('up', '-d', 'dispatcher', 'worker')
|
||||
|
||||
def get_state():
|
||||
return c.call('GET', '/api/v1/attachments/' + ident, actor=alice, schema='Attachment')['json']
|
||||
failed = wait_for('permanent failed status', get_state,
|
||||
lambda a: a['status'] == 'failed', seconds=90)
|
||||
check(failed['error_code'] == 'invalid_image',
|
||||
'expected permanent invalid_image, got ' + str(failed['error_code']))
|
||||
check(job_query(ident, 'status') == 'failed', 'processing job must fail')
|
||||
check(job_query(ident, 'attempt') == '1', 'permanent error must not retry')
|
||||
check(scalar("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % ident) == '0',
|
||||
'failed job created variants')
|
||||
print('Permanent invalid_image confirmed; no retry and no variant rows', flush=True)
|
||||
# Repair only the generated synthetic row before explicit operator replay.
|
||||
restored = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='failed' RETURNING id" %
|
||||
(original_sha, ident))
|
||||
check(ident in restored, 'synthetic metadata could not be restored')
|
||||
changed = False
|
||||
compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident)
|
||||
verify_ready(c, alice, att, original, seconds=180)
|
||||
audit = scalar("SELECT count(*) FROM job_replay_audit WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % ident)
|
||||
check(audit == '1', 'operator replay audit count incorrect: ' + audit)
|
||||
check(job_query(ident, 'status') == 'ready', 'replayed job not ready')
|
||||
attempt = job_query(ident, 'attempt')
|
||||
check(attempt == '1', 'replay did not reset attempt: ' + attempt)
|
||||
invalid = compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs',
|
||||
'replay', '--id', ident, expected=(1,))
|
||||
print('PASS: failed-replay; permanent failure → metadata repaired → replay → ready; audit=1; replay of ready rejected')
|
||||
finally:
|
||||
if changed:
|
||||
# Never leave the synthetic attachment with altered metadata.
|
||||
pg("UPDATE attachments SET sha256='%s' WHERE id='%s'" % (original_sha, ident))
|
||||
|
||||
|
||||
def staging_cleanup(c):
|
||||
compose('up', '-d', 'dispatcher', 'worker')
|
||||
alice, _bob, att, original = make_upload(c)
|
||||
verify_ready(c, alice, att, original, seconds=180)
|
||||
wait_staging_cleared(att['id'], seconds=30)
|
||||
verify_staging_absent(att['id'])
|
||||
print('PASS: staging-cleanup; ready, DB staging_key=NULL, original staging object absent in S3')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--scenario', required=True,
|
||||
choices=['broker-persistence', 'failed-replay', 'staging-cleanup'])
|
||||
args = parser.parse_args()
|
||||
if os.getenv('LAB5_ALLOW_FAILURE_TEST') != 'yes':
|
||||
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes; run only on your isolated Lab5 stack')
|
||||
c = Client('http://localhost:8080')
|
||||
try:
|
||||
wait_api()
|
||||
if args.scenario == 'broker-persistence':
|
||||
broker_persistence(c)
|
||||
elif args.scenario == 'failed-replay':
|
||||
failed_replay(c)
|
||||
else:
|
||||
staging_cleanup(c)
|
||||
finally:
|
||||
# Restore normal services even when an assertion fails. NO volume deletion.
|
||||
try:
|
||||
compose('up', '-d', '--wait', 'broker')
|
||||
compose('up', '-d', 'dispatcher', 'worker')
|
||||
except Failure:
|
||||
print('WARNING: inspect docker compose ps and restore dispatcher/worker manually')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit operator evidence for Lab5 queue durability and failure windows.
|
||||
|
||||
Runs only on an isolated lab5 Compose stack. Never resets named volumes. The test
|
||||
preserves user/session secrets strictly in process memory and prints no tokens.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from client import Client, Failure, check, run
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, expected=(0,)):
|
||||
p = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=240)
|
||||
if p.returncode not in expected:
|
||||
raise Failure('docker compose command failed: %s (exit %s); inspect docker compose ps/logs'
|
||||
% (' '.join(args[:2]), p.returncode))
|
||||
return p
|
||||
|
||||
|
||||
def wait_api(timeout=60):
|
||||
import urllib.request
|
||||
deadline=time.monotonic()+timeout
|
||||
while time.monotonic()<deadline:
|
||||
try:
|
||||
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2) as r:
|
||||
if r.status==200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(.5)
|
||||
raise Failure('API did not become ready')
|
||||
|
||||
|
||||
def make_queued(client):
|
||||
alice,bob,eve,chat=client.bootstrap()
|
||||
att,original=client.attachment(alice,chat)
|
||||
check(att['status']=='queued','202 must return queued snapshot')
|
||||
c=client.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=alice)
|
||||
attached=client.message(alice,chat,'',attachments=[att['id']])['json']
|
||||
check(attached['attachment_ids']==[att['id']],'queued attachment cannot be linked')
|
||||
return alice,bob,att,original
|
||||
|
||||
|
||||
def verify_ready(client,actor,att,original,seconds=180):
|
||||
r=client.wait_ready(actor,att,wait_seconds=seconds)
|
||||
check(len(r['variants']) == 3, 'expected exactly three variants, no duplicate entries')
|
||||
variants={v['name']:v for v in r['variants']}
|
||||
check(set(variants)=={'thumbnail','crop','optimized'},'missing/extra variants')
|
||||
check((variants['thumbnail']['width'],variants['thumbnail']['height'])==(256,160), 'thumbnail dimensions')
|
||||
check((variants['crop']['width'],variants['crop']['height'])==(128,128), 'crop dimensions')
|
||||
raw=client.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)
|
||||
check(raw['bytes']==original,'original changed across asynchronous pipeline')
|
||||
for name in ('thumbnail','crop','optimized'):
|
||||
v=variants[name]
|
||||
response=client.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor)
|
||||
check(len(response['bytes'])==v['size_bytes'] and hashlib.sha256(response['bytes']).hexdigest()==v['sha256'],
|
||||
'variant checksum mismatch: '+name)
|
||||
return r
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--scenario',choices=['paused','outbox-crash','worker-crash'],required=True)
|
||||
args=parser.parse_args()
|
||||
if os.getenv('LAB5_ALLOW_FAILURE_TEST')!='yes':
|
||||
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes in PowerShell to run destructive failure test')
|
||||
c=Client('http://localhost:8080')
|
||||
# All tests make a fresh attachment while dispatcher/worker are stopped.
|
||||
compose('stop','worker','dispatcher')
|
||||
try:
|
||||
alice,bob,att,original=make_queued(c)
|
||||
print('Created durable queued attachment',att['id'],flush=True)
|
||||
if args.scenario=='paused':
|
||||
compose('restart','api1','api2','broker')
|
||||
compose('up','-d','--wait','broker')
|
||||
wait_api()
|
||||
elif args.scenario=='outbox-crash':
|
||||
# Transaction committed, dispatcher crashes before publish.
|
||||
compose('run','--rm','-e','LAB5_FAULT_BEFORE_PUBLISH=once',
|
||||
'dispatcher',expected=(78,))
|
||||
print('Injected dispatcher exit 78 after committed outbox, before publish')
|
||||
else:
|
||||
compose('up','-d','dispatcher')
|
||||
# Fault after all S3 writes and before PostgreSQL result COMMIT and ack.
|
||||
compose('run','--rm','-e','LAB5_FAULT_AFTER_S3=once',
|
||||
'worker',expected=(77,))
|
||||
print('Injected worker exit 77 after S3 effects, before result/ack')
|
||||
compose('up','-d','dispatcher','worker')
|
||||
wait_api()
|
||||
ready=verify_ready(c,bob,att,original)
|
||||
check(ready['status']=='ready','attachment not ready')
|
||||
print('PASS:',args.scenario,'durable queued->ready; original/3 variants verified; no duplicates')
|
||||
finally:
|
||||
# Restore the normal topology even after an assertion failure.
|
||||
compose('up','-d','dispatcher','worker')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real Python OTLP SDK wire-format test against a local in-process receiver.
|
||||
Requires requirements.txt (does not require Docker or external services).
|
||||
This is not a Collector/backend integration test.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
||||
from opentelemetry.trace import SpanKind
|
||||
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
|
||||
from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest
|
||||
from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest
|
||||
from app import telemetry as ot
|
||||
|
||||
received={k:[] for k in ('/v1/traces','/v1/logs','/v1/metrics')}
|
||||
class Receiver(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
data=self.rfile.read(int(self.headers['Content-Length']))
|
||||
received[self.path].append(data)
|
||||
self.send_response(200);self.send_header('Content-Type','application/x-protobuf');self.send_header('Content-Length','0');self.end_headers()
|
||||
def log_message(self,*args): pass
|
||||
|
||||
def main():
|
||||
server=ThreadingHTTPServer(('127.0.0.1',0),Receiver)
|
||||
threading.Thread(target=server.serve_forever,daemon=True).start()
|
||||
os.environ['OTEL_ENABLED']='yes'
|
||||
os.environ['OTEL_EXPORTER_OTLP_ENDPOINT']='http://127.0.0.1:'+str(server.server_port)
|
||||
ot.init('worker')
|
||||
with ot.span('POST /api/v1/chats/{chat_id}/attachments',kind=SpanKind.SERVER) as root:
|
||||
stored=ot.inject(); trace_id=ot.trace_id()
|
||||
with ot.span('rabbitmq.publish',kind=SpanKind.PRODUCER,context=ot.extract(stored)) as producer:
|
||||
sent=ot.inject()
|
||||
with ot.span('rabbitmq.consume',kind=SpanKind.CONSUMER,context=ot.extract(sent)) as consumer:
|
||||
ot.event('job.ready',job_id='synthetic-id',attempt=2)
|
||||
ot.count('lab_jobs',{'job_type':'attachment.process.v1','outcome':'ready'})
|
||||
ot.observe('lab_job_duration',2.5,{'job_type':'attachment.process.v1','outcome':'ready'})
|
||||
ot.observe('lab_ready_duration',5.0,{'job_type':'attachment.process.v1'})
|
||||
class FakeS3:
|
||||
def put_object(self,**kw): return True
|
||||
ot.S3Tracing(FakeS3()).put_object(Key='SECRET_FILENAME_SENTINEL',Body=b'SECRET_CONTENT_SENTINEL')
|
||||
ot.flush()
|
||||
for provider in reversed(ot.providers): provider.shutdown()
|
||||
server.shutdown()
|
||||
spans=[s for raw in received['/v1/traces'] for r in ExportTraceServiceRequest.FromString(raw).resource_spans for scope in r.scope_spans for s in scope.spans]
|
||||
byname={s.name:s for s in spans}
|
||||
assert byname['rabbitmq.publish'].parent_span_id==root.get_span_context().span_id.to_bytes(8,'big')
|
||||
assert byname['rabbitmq.consume'].parent_span_id==producer.get_span_context().span_id.to_bytes(8,'big')
|
||||
assert all(s.trace_id.hex()==trace_id for s in spans)
|
||||
assert 's3.put_object' in byname
|
||||
logs=[l for raw in received['/v1/logs'] for r in ExportLogsServiceRequest.FromString(raw).resource_logs for scope in r.scope_logs for l in scope.log_records]
|
||||
assert any(l.trace_id.hex()==trace_id and l.span_id==consumer.get_span_context().span_id.to_bytes(8,'big') and l.body.string_value=='job.ready' for l in logs)
|
||||
points=[m for raw in received['/v1/metrics'] for r in ExportMetricsServiceRequest.FromString(raw).resource_metrics for scope in r.scope_metrics for m in scope.metrics]
|
||||
assert any(m.name=='lab_jobs' and m.sum.data_points[0].as_int==1 for m in points)
|
||||
assert any(m.name=='lab_job_duration' and list(m.histogram.data_points[0].explicit_bounds)==list(ot.BUCKETS) for m in points)
|
||||
for raw in sum(received.values(),[]): assert b'SECRET_FILENAME_SENTINEL' not in raw and b'SECRET_CONTENT_SENTINEL' not in raw
|
||||
print('PASS: real OTLP trace/log/metric encoding, parent links, histogram buckets, safe S3 attributes')
|
||||
|
||||
if __name__=='__main__': main()
|
||||
Reference in New Issue
Block a user