Implement lab6 observability with OpenTelemetry and Grafana

This commit is contained in:
ferraronp
2026-11-17 19:14:43 +03:00
parent 470d9d6f1e
commit e636d4cc76
68 changed files with 5863 additions and 47 deletions
+23 -25
View File
@@ -1,30 +1,28 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# Copy to .env and REPLACE all sample secrets with random unique values.
# For local synthetic data only. Never commit .env.
POSTGRES_DB=messenger
POSTGRES_USER=lab
POSTGRES_PASSWORD=REPLACE_WITH_RANDOM_LOCAL_PASSWORD
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
DB_POOL_MAX=8
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
S3_ADMIN_ACCESS_KEY=adminlab4
S3_ADMIN_SECRET_KEY=REPLACE_WITH_DIFFERENT_RANDOM_ADMIN_SECRET
S3_ACCESS_KEY=messengerapi
S3_SECRET_KEY=REPLACE_WITH_DIFFERENT_RANDOM_API_SECRET
# RabbitMQ persistent queue (different from session Redis).
BROKER_USER=labworker
BROKER_PASSWORD=REPLACE_WITH_RANDOM_LOCAL_BROKER_PASSWORD
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
JOB_BACKLOG_LIMIT=1000
WORKER_PREFETCH=1
# Only for deliberate crash experiment: once. Keep off for normal operation.
LAB5_FAULT_AFTER_S3=off
# Observability (loopback-only Grafana / diagnostics). No public OTLP ports.
GRAFANA_ADMIN_PASSWORD=REPLACE_WITH_DIFFERENT_RANDOM_GRAFANA_PASSWORD
OTEL_ENABLED=yes
LAB6_FAULT_RETRY_ONCE=off
+10
View File
@@ -0,0 +1,10 @@
FROM python:3.12.10-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
WORKDIR /srv
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt && groupadd --gid 999 app && useradd -m --uid 999 --gid app app
COPY app ./app
COPY db ./db
USER app
EXPOSE 8080
CMD ["python", "-m", "app"]
+1 -1
View File
@@ -1,4 +1,4 @@
PYTHON ?= python3
PYTHON ?= python
BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
+192 -21
View File
@@ -1,37 +1,208 @@
# Отчёт — лабораторная 6
# Отчёт — лабораторная 6. Наблюдаемость
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор / группа: Авзалов Артур Марсович / БПИ2301.
- Целевая оценка: 4; трек 5 не заявлен.
- Статус: кандидат на 4. Реализация проверена на Windows/Docker Desktop 2026-11-17: все шесть сценариев наблюдаемости, публичный smoke, ряд регрессионных тестов Lab5 и визуализация Grafana прошли. Целевая оценка 4; окончательную оценку определяет преподаватель.
- Commit/PR: указывается после публикации.
- ИИ: ChatGPT помог проектировать, реализовывать инструментацию, инфраструктуру, тесты и документацию. Результаты ниже получены при запуске именно Lab6; ChatGPT помог исправить диагностический тест backend-down.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Docker Desktop (Linux containers, Compose V2), Python 3.9+ для публичных HTTP-тестов; Python 3.12 рекомендуется для локальных SDK/unit-проверок. Приложение использует Python 3.12.10. Желательно выделить Docker 6–8 GiB RAM с запасом. Перед запуском остановить Lab5 и другие стенды, занимающие 8080/3000/9090/3100/3200. Не удалять их volumes.
## Архитектура и решения
PowerShell (корень проекта):
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```powershell
Copy-Item .env.example .env
notepad .env
$env:PYTHONUTF8 = "1"
# Заменить ВСЕ REPLACE_WITH_... на разные случайные значения.
# Для паролей в DSN выбрать hex/URL-safe значения, не вставлять секреты в отчёт.
# OTEL_ENABLED=yes, LAB5_FAULT_AFTER_S3=off, LAB6_FAULT_RETRY_ONCE=off.
docker compose --env-file .env config --quiet
docker compose --env-file .env run --rm --no-deps otel-collector validate --config=/etc/otelcol/config.yaml
docker compose --env-file .env run --rm --no-deps --entrypoint /bin/promtool prometheus check config /etc/prometheus/prometheus.yml
docker compose --env-file .env up -d --build --wait
docker compose --env-file .env ps
python scripts/wait_ready.py
make check
make test
```
`--wait` для backend без healthcheck подтверждает только запущенный контейнер, не готовность ingestion. Дождаться `/-/ready` Prometheus, `/ready` Loki/Tempo, `/api/health` Grafana, `up` для Collector; расширенные тесты делают это автоматически. Grafana: http://127.0.0.1:3000, пользователь `admin`, пароль `GRAFANA_ADMIN_PASSWORD` из локального `.env` (задаётся при первом создании volume). Dashboard MTUCI → Lab6 | Messenger observability (`/d/lab6-messenger/`). Prometheus alerts: http://127.0.0.1:9090/alerts. Порты диагностики доступны только на loopback; никакого наружного OTLP или прямых API-реплик.
Для локальных unit/SDK-проверок в отдельном Python 3.12 venv:
```powershell
python -m pip install -r requirements.txt
python scripts/check_lab6.py
python tests/jobs_sql_contract.py
python tests/own_unit.py
python tests/telemetry_unit.py
# Необязательная полная OpenAPI-валидация (отдельный dev requirement):
python -m pip install -r requirements-dev.txt
make validate
```
Безопасная остановка: `docker compose --env-file .env down` (named volumes сохраняются). Только намеренный сброс СВОЕГО учебного стенда: `docker compose --env-file .env down -v` (удаляет и бизнес-данные, и историю телеметрии, не использовать для тестов). `.env`, реальные credentials и данные вне Git. Инициализация `migrate` транзакционна и идемпотентна; `s3-policy` готовит приватные credentials/policy, `s3-bootstrap` bucket, `telemetry-init` устанавливает права директорий named volumes. RabbitMQ queue создаётся приложением с durable/publisher confirms. Provisioning Grafana полностью из Git.
## Архитектура и перенос Lab5
```text
HTTP -> HAProxy -> API1/API2 -> PostgreSQL + Redis + SeaweedFS
-> outbox -> dispatcher -> RabbitMQ -> worker
worker -> S3 + DB ready
API / dispatcher / worker (включая дочерний image process)
| OTLP HTTP/protobuf :4318, внутри telemetry
v
OTel Collector
| metrics /metrics :8889 <- Prometheus scrape
| logs native OTLP HTTP -> Loki
| traces OTLP gRPC -> Tempo
v
Grafana (запросы к трём независимым stores)
```
Бизнес-контракт Lab6 и исходные публичные тесты не изменены; добавляется необязательный диагностический response header `X-Trace-Id` на business HTTP при активной телеметрии. Метаданные, состояние job/outbox и идемпотентность живут в PostgreSQL; сессии в Redis AOF; оригиналы/варианты в S3; RabbitMQ доставляет задачи at-least-once. SDK и Collector не участвуют в транзакциях приложения. Worker сохраняет исходный timeout/lease и отдельный process для Pillow; таймаут/ошибки приводят к durable retry через Outbox. Изменения в `app/jobs.py`, `app/main.py`, `app/storage.py`, `app/attachments.py`, `app/image_worker.py` касаются инструментации, измерения backlog и обработки DB statement timeout; dispatcher обслуживает heartbeat даже в idle. Миграции 001–004 сохранены. Новых бизнес-таблиц не нужно: контракт job допускает optional `traceparent`, `tracestate`, они сохраняются в JSONB outbox.
### Роли / версии / данные / доступ
| Service | Версия (образ) | Данные, сеть, readiness |
| --- | --- | --- |
| api1/api2 | Python 3.12.10-slim-bookworm, FastAPI 0.115.12 | stateless, frontend/storage/telemetry; `/health/ready`, не опубликованы напрямую |
| lb | HAProxy 3.1.7-alpine | frontend, 127.0.0.1:8080; readiness upstream |
| postgres | 16.10-bookworm | postgres_data, storage, pg_isready |
| redis | 7.4.5-alpine | redis_data, storage, redis-cli ping, AOF |
| s3 | SeaweedFS 4.53 | s3_data, s3_config, storage; bootstrap + API head_bucket |
| broker | RabbitMQ 4.1.8-management-alpine | rabbitmq_data, storage, diagnostics ping |
| dispatcher/worker | Python 3.12.10, Pika 1.3.2, Pillow 11.3.0 | storage/telemetry, restart policy; live fault tests |
| otel-collector | contrib 0.123.0 | memory-only bounded batches, telemetry; scrape :8888/:8889 |
| prometheus | v3.2.1 | prometheus_data, telemetry, loopback 9090; /-/ready |
| loki | 3.4.2 | loki_data, TSDB v13 filesystem, loopback 3100; /ready |
| tempo | 2.7.2 | tempo_data, local blocks + WAL, loopback 3200; /ready |
| grafana | 11.6.0 | grafana_data + Git provisioning, loopback 3000; /api/health |
| telemetry-init | BusyBox 1.37.0 | one-shot chown только корней named volumes, сеть отсутствует |
API 384 MiB/0.75 CPU на реплику, worker 768 MiB/1 CPU, dispatcher 192 MiB/0.5 CPU, Collector 256 MiB/0.75 CPU, Prometheus 384 MiB, Loki/Tempo 512 MiB каждый, Grafana 384 MiB. Реальные CPU/RAM замеряются на стенде; лимиты не измерения. Prometheus retention 72h/512MB, Loki и Tempo retention 72h (асинхронная очистка backend, не жёсткий disk quota). Все конфиги `infra/otel/` монтируются read-only. `.env.example` содержит логические настройки runtime, Compose отображает POSTGRES_* в DATABASE_URL, Redis URL фиксируется внутренней сетью, S3_* в credentials, BROKER_* в AMQP credentials. Локальный файл `.env` не коммитится; секреты хранятся в окружении контейнеров учебного стенда, не production secret manager. `STAGING_RETENTION_SECONDS` при отсутствии настройки приложения равен 86400. `WORKER_PREFETCH=1`, один активный job на worker. `OTEL_ENABLED=yes` включает SDK; endpoint `http://otel-collector:4318` задаётся Compose. `GRAFANA_ADMIN_PASSWORD` только для локального интерфейса. `LAB6_FAULT_RETRY_ONCE` — операторская настройка, по умолчанию off (не API).
## Сигналы, запросы и единицы
SDK OTel API/SDK/OTLP HTTP 1.42.1, явная инструментация `app/telemetry.py`. Стабильные локальные names и ограниченные semantic attributes вместо автоматического capture. `service.name=messenger-api|messenger-dispatcher|messenger-worker`, `service.version=1.6.0`, `service.instance.id=api1|api2|container hostname`, `deployment.environment.name=lab`; Prometheus exporter переводит точки в подчёркивания, добавляет суффиксы секунд/байт/total. Все histogram buckets одинаковы на репликах: 0.005…600 s. Health измеряется, но исключён из business SLI и трассировки (чтобы probe не загрязнял трассы). HTTP duration измеряется до получения заголовков ответа; потоковая передача content целиком не измеряется. Не именуем p99 средним.
| Сигнал / Prometheus имя | Выражение / единица |
| --- | --- |
| HTTP throughput и 5xx | `sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))`, фильтр `status_class="5xx"` для ошибок; req/s |
| p50/p95/p99 | `histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))`, аналогично 0.5/0.99; секунды |
| Job outcomes | `sum by(outcome)(rate(lab_jobs_total[30s]))`, attempts/s; ready/retry/failed/stale/requeue; hard kill до метрики не считается завершённым attempt |
| Worker processing | `histogram_quantile(0.95,sum by(le)(rate(lab_job_duration_seconds_bucket[30s])))`, секунды claimed attempt |
| Time to ready | `histogram_quantile(0.95,sum by(le)(rate(lab_ready_duration_seconds_bucket[30s])))`, секунды от `processing_jobs.created_at` (время транзакции durable acceptance, близкое, но не точно равное моменту COMMIT) до commit ready, включает retry/backlog |
| Durable backlog | `max(lab_queue_depth)`: число jobs в статусе queued, включая неопубликованные и delayed retry |
| Broker queue | `max(lab_broker_ready)`: ready messages RabbitMQ через passive declare; `lab_broker_consumers` — число consumers; не включает unacked и не равно DB backlog |
| Oldest age | `max(lab_queue_oldest_age_seconds)`: now−created_at старейшей queued job; секунды; нет queued = 0 |
| DB pool | `lab_db_pool_used`, `lab_db_pool_waiting`, `lab_db_pool_capacity` по service_instance_id; connections/requests; used = size−available (включая reserved/создаваемые соединения) |
| Worker saturation | `sum(lab_worker_busy)/clamp_min(sum(lab_worker_capacity),1)`; ratio, active/capacity. Один job на процесс. При отсутствии worker ряд исчезает по expiration, это не доказательство idle |
| CPU/RAM | `lab_container_cpu_cores` (CPU seconds / wall seconds), `lab_container_memory_bytes` (cgroup v2 usage, fallback v1 CPU / RSS дерева процессов) |
| Collector internal | `otelcol_exporter_queue_size`, `otelcol_exporter_queue_capacity`, `otelcol_exporter_send_failed_spans[_total]`, `otelcol_exporter_send_failed_log_records[_total]`, `otelcol_process_memory_rss` / `otelcol_process_memory_rss_bytes`; учитывать текущую нормализацию выбранной версии |
`method`, route template, `status_class`, `job_type`, `outcome`, `queue` — ограниченный набор label values, `service_instance_id` ограничен числом экземпляров за retention; нет raw URL, filename, user/chat/message/attachment/request/job ID. Метрики cumulative, Collector выставляет Prometheus exposition. `metric_expiration: 15s` ограничивает stale gauges (нет данных при недоступной БД, а не искусственный 0). Percentiles строятся по сумме histogram buckets всех реплик (не усреднение p95). При отсутствии событий rate/quantile может быть NaN: требуется свежая нагрузка, NaN не заменяем нулём искусственно.
### Сквозная трасса и логи
1. API создаёт реальный SERVER span на business route; `db.transaction` и `s3.upload_fileobj` видны как dependency spans. Request ID остаётся прежним; `http.completed` содержит request_id, status_code, method, route, trace/span ID, timestamp и severity.
2. Во время транзакции job/outbox `job_payload()` добавляет W3C `traceparent`/`tracestate`. Dispatcher извлекает их; создаёт PRODUCER span и внедряет **свой** context в сообщение RabbitMQ. Outbox хранит исходный контекст. На retry вызов из CONSUMER сохраняет его context; lease rescue/replay без активного span использует первый сохранённый context.
3. Worker извлекает context, CONSUMER является child producer; проверяет job/lease в БД. `image.subprocess` передаёт context дочернему процессу в env без секретов телеметрии; видны `image.transform`, `db.load_staging_metadata`, `s3.get_object`, `s3.put_object`, `s3.copy_object`, затем `db.publish_result`. Worker завершает `job.ready` или `job.retry`/`job.failed`, ack после durable результата/retry. В случае hard kill последний span может не успеть экспортироваться; retry продолжает trace.
4. В Grafana Explore → Tempo по `X-Trace-Id` / trace_id; открываем span; Loki: `{service_name="messenger-api"} | trace_id="<trace_id>" | request_id="<request_id>"`, worker `{service_name="messenger-worker"} | trace_id="<trace_id>"`. Поля `trace_id`, `span_id`, `event` — OTLP structured metadata; тела записей фиксированные технические события. Технический job_id только в trace/log для конкретного учебного задания; SQL text, exception messages/stack traces, raw URLs, Authorization, passwords, session tokens, тело сообщений и файлы не передаются SDK. Отключён uvicorn access log, только отдельный безопасный logger передаётся в OTLP. Это проверкой синтетических sentinel полей подтверждено локально, не исчерпывающий security-аудит.
### SLI / SLO и alerts
SLI = 1−(число business HTTP 5xx / число всех завершённых business HTTP responses), health исключён, 4xx клиента не считаются отказом сервера. Окно 5 минут; **SLO >= 99%** (учебный, выбран нами, не обещание SLA). Dashboard SLI: `1 - ((sum(increase(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[5m])) or vector(0)) / sum(increase(lab_http_requests_total{route=~"/api/.*"}[5m])))`. Если нет трафика — нет измеримого SLI. SLI не учитывает отказ между клиентом и HAProxy либо ответы самого HAProxy, поскольку SDK в API; для production нужны LB/probe metrics. Время до ready отдельно измеряется histogram; успешные ready не учитывают навсегда failed, поэтому не подменяем им SLI доступности.
Правила `infra/otel/prometheus/alerts.yml`: `LabAPIHighErrorRate` >5% business 5xx за 30 s, `for: 9s`; `LabQueueLag` age >15 s и depth>0, `for: 12s`. Это локальные diagnostic alert thresholds, не burn-rate track 5. Prometheus evaluation/scrape 3s; firing/pending/inactive (inactive после firing = resolved) в Prometheus UI/API, Grafana dashboard `ALERTS`. Проверка искусственными отказами — ниже; внешней доставки нет.
### Backpressure и сбой телеметрии
SDK async BatchSpan/BatchLogRecordProcessor queue 512 items/64 batch, HTTP exporter 2s timeout; metric reader 3s interval/2s export timeout. Не ждём backend для HTTP/normal commit; дочерний process ограниченно flush'ит (добавляет latency, остаётся внутри JOB_TIMEOUT), потеря телеметрии допустима. Collector memory_limiter 180 MiB (spike 40 MiB), container limit 256 MiB, GOMEMLIMIT 180MiB. Batch 128/256 items, timeout 1s; каждый Tempo/Loki exporter queue 64 batches, один consumer, request timeout 3s, retry до 10s. При переполнении/отказе telemetry может быть потеряна; нет бесконечной durable очереди и нет гарантий delivery telemetry. Prometheus scrape хранит history самостоятельно. `backend-down` проверяет успешные бизнес-записи/обработку изображения при остановленном Tempo, рост failed exports, sampled RSS и queue size, отсутствие OOM/restart и свежий trace после восстановления. Failed export counter отражает неуспешный экспорт Collector (не точное число всех потерянных spans SDK; повторные попытки и обрывы могут влиять). Измерение на коротком сценарии не доказательство бессрочной нагрузки.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Локальные проверки (в среде подготовки, 2026-11-17) см. [evidence/offline_validation.md](evidence/offline_validation.md). Фактические результаты Docker-прогона на Windows 2026-11-17 сведены в [evidence/verified_results_2026-11-17.md](evidence/verified_results_2026-11-17.md). JSON-файлы сформированы тестами на стенде; скриншоты Grafana — в `evidence/grafana/`. Неповторённые проверки отмечены отдельно. Скрипты добавлены отдельно, исходные `tests/smoke.py`, `client.py`, `load.py`, `resilience.py`, fixtures и OpenAPI сохранены byte-for-byte.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий | Команда/сценарий | Результат / evidence |
| --- | --- | --- |
| Структура, SQL, SDK | `make check`, `python scripts/check_lab6.py`, `python tests/jobs_sql_contract.py`, `python tests/telemetry_unit.py`, `python tests/own_unit.py` | локально PASS; SDK wire-format test не проверяет Collector/backend |
| Основной API и регрессия Lab5 | `make test`, `attachments_negative`, `idempotency_parallel`, `own_unit`, `pipeline_resilience` (paused/outbox-crash/worker-crash) | PASS на Lab6; отдельные `resilience.py` и `pipeline_extended.py` в данном прогоне не подтверждены |
| Все три stores, dashboard/datasources, метрики + низкая кардинальность | `observability_live.py --scenario metrics` | PASS: ожидаемые ряды и отсутствие запрещённых ID labels; `evidence/lab6-metrics.json` |
| Upload/producer/consumer/retry, лог+trace, S3+DB result | `--scenario trace-retry` | PASS: API/producer/consumer/S3/DB с retry, связанные логи; `evidence/lab6-trace-retry.json` |
| Queue age / depth / SLO & alert | `--scenario queue-lag` | PASS: возраст 27.574387 s; queued=1; firing → ready → resolved; `evidence/lab6-queue-lag.json` |
| DB latency / 5xx alert | `--scenario db-error` | PASS: QueryCanceled/503, error ratio 1.0; firing → resolved; `evidence/lab6-db-error.json` |
| Collector ограничен, telemetry backend down | `--scenario backend-down` | PASS после исправления теста: 16 успешных записей, sampled max RSS 200.43 MiB / 256 MiB, queue 18/64, свежая трасса после восстановления; `evidence/lab6-backend-down.json` |
| Сохранение метрик/логов/traces при restart | `--scenario history` | PASS: сохранены метрики, логи, трассы после перезапуска backend; `evidence/lab6-history.json` |
| Нагрузка, percentiles, ресурсы | `tests/load.py`, `attachments_concurrent.py`, dashboard CPU/RAM, PromQL | PASS: 10/10 HTTP 202 → ready, по 3 варианта; acceptance 3.976 RPS, p50 2296.5 ms, max 2484.0 ms, все ready за 36.406 s; `evidence/attachments-concurrent.json`; 16 панелей Grafana показывают данные. Отдельный `tests/load.py` ещё не подтверждён |
## Эксперименты и отказы
### Команды полного прогона на Windows
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Список включает как уже выполненные, так и дополнительные регрессионные проверки. Фактический статус — в сводке доказательств; команды сами по себе не означают PASS.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
```powershell
$env:PYTHONUTF8 = "1"
$env:LAB5_ALLOW_FAILURE_TEST = "yes"
$env:LAB6_ALLOW_FAILURE_TEST = "yes"
# Все fault tests строго последовательно, только на отдельном учебном стенде.
make test
python tests/attachments_negative.py
python tests/idempotency_parallel.py
python tests/own_unit.py
python tests/pipeline_resilience.py --scenario paused
python tests/pipeline_resilience.py --scenario outbox-crash
python tests/pipeline_resilience.py --scenario worker-crash
python tests/pipeline_extended.py --scenario broker-persistence
python tests/pipeline_extended.py --scenario failed-replay
python tests/pipeline_extended.py --scenario staging-cleanup
python tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
python tests/attachments_concurrent.py --concurrency 10 --output evidence/attachments-concurrent.json
python tests/observability_live.py --scenario metrics
python tests/observability_live.py --scenario trace-retry
python tests/observability_live.py --scenario queue-lag
python tests/observability_live.py --scenario db-error
python tests/observability_live.py --scenario backend-down
python tests/observability_live.py --scenario history
python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
docker compose --env-file .env up -d --no-deps --wait api1 api2
make test
```
На Linux соответствующие hooks `restart.sh`, `stop-one.sh` (executable). Не запускать сценарии одновременно. Fault scripts управляют реальными контейнерами, не удаляют volumes. `trace-retry` временно пересоздаёт worker с operator-only fault once, затем восстанавливает off. `db-error` удерживает lock не более 60s после захвата; освобождение в конце транзакции; не применять на production. `history` перезапускает telemetry stores с named volumes. При ошибке смотреть `docker compose logs --tail=80 <service>`, не публиковать secrets. `make check` не заменяет runtime. Если Docker не стартует, не менять публичные тесты ради PASS.
## Инциденты и runbook
**Инцидент 1, гипотеза: остановлен worker.** `queue-lag` принимает синтетический upload при остановленном worker. Через метрики backlog/oldest age определяем растущую очередь, сравниваем broker ready/consumer count; API может продолжать отвечать 202, CPU worker отсутствует, DB pool нормальный. `LabQueueLag` firing; trace upload → publish есть, consume отсутствует. Запустить worker и дождаться ready, depth=0, alert inactive. Время обнаружения/возраст/глубина — в JSON результата. Факт (17.11): `queue-lag` PASS, age 27.574387 s, queued=1 при firing; после запуска worker задание стало ready, alert resolved. См. `evidence/lab6-queue-lag.json` и [графики очереди](evidence/grafana/02-worker-and-queue.png).
**Инцидент 2, гипотеза: DB заблокирована, не медленный handler.** `db-error` удерживает кратковременный ACCESS EXCLUSIVE LOCK на `users` в отдельной psql транзакции, health SELECT 1 остаётся рабочим. Создаётся несколько business GET `/api/v1/users/me`, которые получают 503 после database statement timeout ~5s или pool timeout. В Grafana: HTTP errors/p95 растут, `db.transaction` занимает основное время и имеет error.type QueryCanceled, связанный http.completed лог с request_id; отличить от долгого обработчика с нормальными dependency spans. `LabAPIHighErrorRate` firing; после COMMIT lock запросы снова 200, error rate падает и alert inactive. Факт (17.11): `db-error` PASS, QueryCanceled и HTTP 503 обнаружены, error ratio=1.0 при firing, после снятия блокировки alert resolved. См. `evidence/lab6-db-error.json` и [графики API](evidence/grafana/01-api-and-jobs.png).
**Инцидент 3: Tempo недоступна.** `backend-down` останавливает Tempo, посылает synthetic messages и image upload. Prometheus продолжает считать business telemetry; Collector failed spans/queue/RSS показывают сбой экспорта, API/worker продолжают обработку. После возврата Tempo создаётся новая трасса, старые могли потеряться. Факт (17.11): 16 успешных записей, обработка изображения сохранена; RSS Collector до отказа 197.35 MiB, наблюдаемый максимум 200.43 MiB при лимите 256 MiB; очередь выросла с 0/64 до 18/64. Ошибки экспорта наблюдались, свежая трасса после восстановления доступна. См. `evidence/lab6-backend-down.json` и [графики Collector](evidence/grafana/04-collector-alerts-slo.png). Первоначальный тест завершился на проверке RSS; исправлен только тест (поддержка варианта имени метрики с `_bytes`, строгая проверка очереди и диагностика лимитов); последующий живой прогон PASS.
Для нагрузки фиксировать hardware/выделенные Docker CPU/RAM, прогрев после `up`, дату, commit, размер файла 111817 B, длительность 30s и concurrency 4 для POST messages, concurrency 10 для upload; сохранить `load.json`, `attachments-concurrent.json`, показатели p50/p95/p99 из dashboard, CPU/RAM через PromQL/периодические замеры `docker stats`. Closed-loop нагрузки не доказывают максимальную capacity; p95 из histogram приблизителен. В `evidence/` хранить только обезличенные summary и технические IDs синтетических trace, не raw secrets/requests. Скриншоты Grafana/alerts полезны как дополнение, не вместо stored results.
### Визуальная проверка Grafana (17.11.2026)
Все 16 панелей на четырёх приложенных скриншотах содержат данные реального запуска. Снимки показывают часовой интервал с тестовыми инцидентами и последующей нагрузкой; точные значения и состояния берутся из результатов сценариев, визуальные оценки округлены.
| Доказательство | Что видно |
| --- | --- |
| [API и jobs](evidence/grafana/01-api-and-jobs.png) | Throughput, всплеск доли 5xx до 100%, p95/p99 около 10 s в момент сбоя, job outcomes |
| [Worker и очередь](evidence/grafana/02-worker-and-queue.png) | Processing duration, time to ready (с учётом очереди), backlog, пик возраста около 27 s |
| [Pool и ресурсы](evidence/grafana/03-pool-and-resources.png) | DB pool capacity 8 на реплику, worker saturation достигает 100%, CPU и sampled memory по экземплярам |
| [Collector, alerts, SLO](evidence/grafana/04-collector-alerts-slo.png) | Очередь экспорта Tempo при ограничении 64, failed exports, firing/pending, падение SLI ниже 99% и последующее восстановление |
Для показа связи лога и трассы в Grafana Explore используйте сохранённый trace ID `4d8711b71965876a59aa5594ad0d51a0` из `trace-retry`, пока данные доступны по retention. При истечении retention повторите сценарий. PNG подтверждают визуализацию метрик; связанные логи и spans подтверждены тестом `trace-retry`, а не отдельными скриншотами Explore.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Один Docker-host и одиночные PG, Redis, S3, RabbitMQ, LB, Collector и telemetry backends. При их потере не заявляем HA. Бизнес-данные и telemetry разделены, at-least-once не exactly-once. Наблюдаемость best-effort: SDK queue/Collector memory pressure, hard crash, offline backend могут приводить к пропускам traces/logs/metrics. Трассируем зависимость до заголовков/вызова S3, не каждую SQL-команду и не весь downloaded stream. No sampling research / burn-rate track. Нагрузочные и fault evidence приведены выше. Замеры ограничены коротким сценарием и 10 одновременными upload; отдельно `tests/load.py`, повторный полный persistence/failover бизнес-стенда и три сценария `pipeline_extended.py` в предоставленном выводе Lab6 отсутствуют. Параметры CPU/RAM Docker-host и полный вывод JSON не предоставлены для этой сводки. Примерные пароли использовались на локальном тестовом стенде; при новом развёртывании заменить их перед запуском, не коммитить `.env`.
## Источники
- [Описание задания](README.md), [контракт observability](contracts/observability.md), [схема job](contracts/jobs.schema.json), [карта курса](COURSE.md).
- [OpenTelemetry Collector configuration](https://opentelemetry.io/docs/collector/configuration/), [internal telemetry](https://opentelemetry.io/docs/collector/internal-telemetry/), [context propagation](https://opentelemetry.io/docs/concepts/context-propagation/), [W3C Trace Context](https://www.w3.org/TR/trace-context/).
- [Prometheus histograms](https://prometheus.io/docs/practices/histograms/), [Loki OTLP ingestion](https://grafana.com/docs/loki/latest/send-data/otel/), [Tempo configuration](https://grafana.com/docs/tempo/latest/configuration/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/).
+1
View File
@@ -0,0 +1 @@
"""Lab 3 messenger API."""
+4
View File
@@ -0,0 +1,4 @@
import os
import uvicorn
if __name__ == "__main__":
uvicorn.run("app.main:app", host="0.0.0.0", port=int(os.getenv("APP_PORT", "8080")), workers=1, access_log=False, timeout_graceful_shutdown=8)
+193
View File
@@ -0,0 +1,193 @@
"""Messenger HTTP handlers; state lives only in PostgreSQL/Redis."""
import hashlib
import json
import re
import uuid
import psycopg
from argon2 import PasswordHasher
from fastapi import APIRouter, Request
from .auth import current_user, hasher, login, logout, public_user
from .core import APIError, invalid, iso, make_cursor, parse_limit, parse_uuid, read_cursor
from .models import CreateChat, CreateMessage, CreateUser
from .storage import db
router = APIRouter()
KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$", re.ASCII)
def actor(req: Request) -> dict:
return getattr(req.state, "actor", None) or current_user(req)
def chat_detail(conn, chat: dict) -> dict:
members = conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position", (chat["id"],)).fetchall()
return {"id": str(chat["id"]), "title": bytes(chat["title"]).decode("utf-8"), "created_at": iso(chat["created_at"]), "member_ids": [str(x["user_id"]) for x in members]}
def message_detail(conn, row: dict) -> dict:
linked = conn.execute("SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", (row["id"],)).fetchall()
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": bytes(row["text"]).decode("utf-8"), "created_at": iso(row["created_at"]), "attachment_ids": [str(x["attachment_id"]) for x in linked]}
def require_chat(conn, chat_id: str, actor_id: str) -> dict:
row = conn.execute("SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id WHERE c.id=%s AND cm.user_id=%s", (chat_id, actor_id)).fetchone()
if row is None:
raise APIError(404, "not_found", "Object not found")
return row
@router.get("/health/live")
def live():
return {"status": "ok"}
@router.get("/health/ready")
def ready():
from .storage import ready as storage_ready
if not storage_ready():
raise APIError(503, "unavailable", "Service not ready")
return {"status": "ok"}
@router.post("/api/v1/users", status_code=201)
def register(body: CreateUser):
hashed = hasher.hash(body.password)
ident = str(uuid.uuid4())
try:
with db() as conn:
row = conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (ident, body.username, body.display_name.encode("utf-8"), hashed)).fetchone()
return public_user(row)
except psycopg.errors.UniqueViolation:
raise APIError(409, "conflict", "Username already exists") from None
@router.post("/api/v1/auth/sessions", status_code=201)
def sessions(req: Request):
return login(req)
@router.delete("/api/v1/auth/sessions/current", status_code=204)
def sessions_delete(req: Request):
logout(req)
return None
@router.get("/api/v1/users/me")
def users_me(req: Request):
return actor(req)
@router.post("/api/v1/chats", status_code=201)
def chats_create(req: Request, body: CreateChat):
ident = actor(req)["id"]
if ident not in body.member_ids:
raise invalid("Current user must belong to the chat")
chat_id = str(uuid.uuid4())
try:
with db() as conn:
with conn.transaction():
# Validate all members; FK constraints are the final authority for races.
found = conn.execute("SELECT id FROM users WHERE id = ANY(%s::uuid[])", (body.member_ids,)).fetchall()
if len(found) != len(body.member_ids):
raise invalid("Unknown member")
row = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (chat_id, body.title.encode("utf-8"))).fetchone()
for pos, member in enumerate(body.member_ids):
conn.execute("INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)", (chat_id, member, pos))
result = chat_detail(conn, row)
return result
except psycopg.errors.ForeignKeyViolation:
raise invalid("Unknown member") from None
@router.get("/api/v1/chats")
def chats_list(req: Request, limit: str | None = None, cursor: str | None = None):
who = actor(req)["id"]
n = parse_limit(limit)
mark = read_cursor(cursor, "chats", who)
with db() as conn:
if mark:
existing = conn.execute("SELECT 1 FROM chats c JOIN chat_members cm ON c.id=cm.chat_id WHERE c.id=%s AND c.created_at=%s AND cm.user_id=%s", (mark[1], mark[0], who)).fetchone()
if not existing:
raise invalid("Invalid cursor")
rows = conn.execute("""
SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id
WHERE cm.user_id=%s AND (%s::timestamptz IS NULL OR (c.created_at,c.id)>(%s::timestamptz,%s::uuid))
ORDER BY c.created_at,c.id LIMIT %s
""", (who, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall()
items = [chat_detail(conn, row) for row in rows[:n]]
more = len(rows) > n
last = rows[n-1] if more else None
return {"items": items, "next_cursor": make_cursor("chats", who, last["created_at"], str(last["id"])) if last else None}
@router.get("/api/v1/chats/{chat_id}")
def chats_get(req: Request, chat_id: str):
who = actor(req)["id"]
ident = parse_uuid(chat_id)
with db() as conn:
row = require_chat(conn, ident, who)
return chat_detail(conn, row)
@router.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def messages_create(req: Request, chat_id: str, body: CreateMessage):
who = actor(req)["id"]
ident = parse_uuid(chat_id)
key = req.headers.get("Idempotency-Key")
if not key or not KEY_RE.fullmatch(key):
raise invalid("Invalid Idempotency-Key")
# Lab3 has only text; canonical JSON avoids unstable formatting/field ordering.
fingerprint = hashlib.sha256(json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
lock_scope = f"{who}:POST:/api/v1/chats/{ident}/messages:{key}"
with db() as conn:
with conn.transaction():
require_chat(conn, ident, who)
# Transaction-scoped advisory lock is GLOBAL across API replicas.
conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (lock_scope,))
previous = conn.execute("""
SELECT idem.body_sha256, m.* FROM message_idempotency idem
JOIN messages m ON m.id=idem.message_id
WHERE idem.actor_id=%s AND idem.chat_id=%s AND idem.method='POST' AND idem.idem_key=%s
""", (who, ident, key)).fetchone()
if previous:
if previous["body_sha256"] != fingerprint:
raise APIError(409, "conflict", "Idempotency key used with another body")
return message_detail(conn, previous)
# 404 for foreign/nonexistent IDs; check BEFORE any message insert.
if body.attachment_ids:
owned = conn.execute("SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])", (ident, body.attachment_ids)).fetchall()
if len(owned) != len(body.attachment_ids):
raise APIError(404, "not_found", "Attachment not found")
if any(item["status"] == "failed" for item in owned):
raise APIError(409, "conflict", "Failed attachment cannot be linked")
message_id = str(uuid.uuid4())
row = conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (message_id, ident, who, body.text.encode("utf-8"))).fetchone()
for pos, attachment_id in enumerate(body.attachment_ids):
conn.execute("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", (message_id, attachment_id, pos))
conn.execute("INSERT INTO message_idempotency(actor_id,chat_id,method,idem_key,body_sha256,message_id) VALUES (%s,%s,'POST',%s,%s,%s)", (who, ident, key, fingerprint, message_id))
return message_detail(conn, row)
@router.get("/api/v1/chats/{chat_id}/messages")
def messages_list(req: Request, chat_id: str, limit: str | None = None, cursor: str | None = None):
who = actor(req)["id"]
ident = parse_uuid(chat_id)
n = parse_limit(limit)
mark = read_cursor(cursor, "messages", ident)
with db() as conn:
require_chat(conn, ident, who)
if mark:
existing = conn.execute("SELECT 1 FROM messages WHERE id=%s AND chat_id=%s AND created_at=%s", (mark[1], ident, mark[0])).fetchone()
if not existing:
raise invalid("Invalid cursor")
rows = conn.execute("""
SELECT * FROM messages WHERE chat_id=%s
AND (%s::timestamptz IS NULL OR (created_at,id)>(%s::timestamptz,%s::uuid))
ORDER BY created_at,id LIMIT %s
""", (ident, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall()
with db() as conn:
items = [message_detail(conn, row) for row in rows[:n]]
last = rows[n-1] if len(rows)>n else None
return {"items": items, "next_cursor": make_cursor("messages", ident, last["created_at"], str(last["id"])) if last else None}
+240
View File
@@ -0,0 +1,240 @@
"""Lab5 attachments: synchronous durable admission, asynchronous image pipeline.
202 means verified bytes were staged to private S3 and metadata + a job + an
outbox event were committed in ONE PostgreSQL transaction. No broker availability
is required for admission; the dispatcher replays unsent intents.
"""
import hashlib
import os
import tempfile
import threading
import uuid
from contextlib import closing
import boto3
from boto3.s3.transfer import TransferConfig
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError
from fastapi import APIRouter, Query, Request
from fastapi.responses import StreamingResponse
from PIL import Image, ImageFile, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from .auth import current_user
from .core import APIError, invalid, iso, parse_uuid
from .storage import db
from . import telemetry as ot
router = APIRouter()
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
SPOOL_MEMORY_BYTES = 1024 * 1024
S3_BUFFER_BYTES = 256 * 1024
DECODE_LIMIT = threading.BoundedSemaphore(1)
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
ImageFile.LOAD_TRUNCATED_IMAGES = False
def s3_client():
return ot.S3Tracing(boto3.client(
"s3", endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
region_name="us-east-1", use_ssl=False,
config=Config(s3={"addressing_style": "path"}, connect_timeout=3,
read_timeout=10, retries={"max_attempts": 1}),
))
def s3_error(exc):
return APIError(503, "unavailable", "Object storage unavailable")
def format_attachment(row, variants=()):
return {
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]), "filename": row["filename"],
"content_type": row["content_type"], "size_bytes": row["size_bytes"],
"sha256": row["sha256"], "status": row["status"],
"variants": [{k: item[k] for k in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
for item in variants] if row["status"] == "ready" else [],
"error_code": row["error_code"] if row["status"] == "failed" else None,
"created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"]),
}
def require_member(conn, chat_id, user_id):
if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone():
raise APIError(404, "not_found", "Object not found")
def attachment_row(conn, attachment_id, user_id):
row = conn.execute("""
SELECT a.* FROM attachments a
JOIN chat_members cm ON cm.chat_id=a.chat_id
WHERE a.id=%s AND cm.user_id=%s
""", (attachment_id, user_id)).fetchone()
if not row:
raise APIError(404, "not_found", "Object not found")
return row
def validate_filename(filename: str):
if not (1 <= len(filename) <= 255) or any(c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename):
raise invalid("Invalid filename")
def validate_image(fileobj, mime: str):
"""Validate MIME, magic, dimensions and complete decode; worker does the resize."""
try:
fileobj.seek(0)
with Image.open(fileobj) as im:
if im.format not in ("PNG", "JPEG"):
raise APIError(415, "unsupported_media_type", "Unsupported image encoding")
actual = "image/png" if im.format == "PNG" else "image/jpeg"
if actual != mime:
raise APIError(415, "unsupported_media_type", "Content-Type does not match image")
if im.width * im.height > MAX_PIXELS:
raise APIError(413, "payload_too_large", "Image dimensions too large")
im.verify()
fileobj.seek(0)
with DECODE_LIMIT:
with Image.open(fileobj) as im:
im.load()
except APIError:
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise APIError(413, "payload_too_large", "Image dimensions too large") from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise APIError(415, "unsupported_media_type", "Invalid image") from None
finally:
fileobj.seek(0)
def _admit_upload(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest):
from .jobs import job_payload
stage_key = f"staging/{attachment_id}/original"
key = f"attachments/{attachment_id}/original"
backup = f"recovery/{attachment_id}/original"
job_id = str(uuid.uuid4())
outbox_id = str(uuid.uuid4())
client = s3_client()
try:
spool.seek(0)
client.upload_fileobj(spool, BUCKET, stage_key,
ExtraArgs={"ContentType": mime},
Config=TransferConfig(use_threads=False, multipart_threshold=16*1024*1024,
io_chunksize=S3_BUFFER_BYTES))
except (BotoCoreError, ClientError, EndpointConnectionError, OSError) as exc:
raise s3_error(exc) from exc
# Atomic DB metadata + intent. If DB commit is ambiguous, do not delete the
# staged object: an accepted task may already exist; operator cleanup handles it.
with db() as conn:
with conn.transaction():
require_member(conn, chat_id, user_id)
# Small educational admission bound; pre-existing queued tasks remain safe.
pending = conn.execute("SELECT count(*) AS c FROM processing_jobs WHERE status IN ('queued','processing')").fetchone()["c"]
if pending >= int(os.getenv("JOB_BACKLOG_LIMIT", "1000")):
raise APIError(429, "rate_limited", "Processing queue is full")
row = conn.execute("""
INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename,
content_type,size_bytes,sha256,status,staging_key,pipeline_version)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,1) RETURNING *
""", (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest, stage_key)).fetchone()
conn.execute("""
INSERT INTO processing_jobs(id,attachment_id,pipeline_version,status,attempt)
VALUES (%s,%s,1,'queued',1)
""", (job_id, attachment_id))
payload = job_payload(job_id, attachment_id, 1, 1)
conn.execute("""
INSERT INTO job_outbox(id,job_id,payload) VALUES (%s,%s,%s::jsonb)
""", (outbox_id, job_id, __import__('json').dumps(payload)))
ot.event("upload.accepted", job_id=job_id)
return format_attachment(row) # snapshot = queued, even if worker is very fast
@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)):
from .api import actor, require_chat
who = actor(req)["id"]
chat_id = parse_uuid(chat_id)
validate_filename(filename)
mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if mime not in ("image/png", "image/jpeg"):
raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
with db() as conn:
require_chat(conn, chat_id, who)
announced = req.headers.get("content-length")
if announced:
try:
if int(announced) > MAX_BYTES:
raise APIError(413, "payload_too_large", "Image too large")
if int(announced) < 0:
raise invalid("Invalid Content-Length")
except ValueError:
raise invalid("Invalid Content-Length") from None
size = 0
digest = hashlib.sha256()
with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool:
async for chunk in req.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, "payload_too_large", "Image too large")
digest.update(chunk)
spool.write(chunk)
if not size:
raise APIError(415, "unsupported_media_type", "Empty image")
await run_in_threadpool(validate_image, spool, mime)
return await run_in_threadpool(_admit_upload, spool, attachment_id=str(uuid.uuid4()),
chat_id=chat_id, user_id=who, filename=filename,
mime=mime, size=size, digest=digest.hexdigest())
@router.get("/api/v1/attachments/{attachment_id}")
def get_attachment(req: Request, attachment_id: str):
who = getattr(req.state, "actor", None) or current_user(req)
with db() as conn:
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
variants = conn.execute("""
SELECT name,content_type,size_bytes,width,height,sha256
FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s
ORDER BY CASE name WHEN 'thumbnail' THEN 0 WHEN 'crop' THEN 1 ELSE 2 END
""", (row["id"], row["pipeline_version"])).fetchall() if row["status"] == "ready" else []
return format_attachment(row, variants)
@router.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(req: Request, attachment_id: str, variant: str = "original"):
who = getattr(req.state, "actor", None) or current_user(req)
with db() as conn:
row = attachment_row(conn, parse_uuid(attachment_id), who["id"])
if row["status"] != "ready":
raise APIError(409, "conflict", "Attachment is not ready")
if variant == "original":
key, mime, size = row["object_key"], row["content_type"], row["size_bytes"]
else:
v = conn.execute("""
SELECT * FROM attachment_variants
WHERE attachment_id=%s AND pipeline_version=%s AND name=%s
""", (row["id"], row["pipeline_version"], variant)).fetchone()
if not v:
raise APIError(404, "not_found", "Object not found")
key, mime, size = v["object_key"], v["content_type"], v["size_bytes"]
client = s3_client()
try:
response = client.get_object(Bucket=BUCKET, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"):
raise APIError(503, "unavailable", "Stored object missing") from exc
raise s3_error(exc) from exc
except BotoCoreError as exc:
raise s3_error(exc) from exc
def iterator():
with closing(response["Body"]) as body:
while chunk := body.read(S3_BUFFER_BYTES):
yield chunk
return StreamingResponse(iterator(), media_type=mime, headers={"Content-Length": str(size)})
+104
View File
@@ -0,0 +1,104 @@
"""HTTP Basic only on login; bearer sessions shared by all API replicas."""
import base64
import binascii
import hashlib
import re
import secrets
import uuid
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError, VerificationError
from fastapi import Request
from redis.exceptions import RedisError
from .core import APIError, SESSION_TTL, iso
from .storage import cache, db
hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
PASSWORD_RE = re.compile(r"^[a-z0-9_]{3,32}$")
def public_user(row: dict):
return {"id": str(row["id"]), "username": row["username"], "display_name": bytes(row["display_name"]).decode("utf-8"), "created_at": iso(row["created_at"])}
def unauthorized() -> APIError:
return APIError(401, "unauthorized", "Unauthorized")
def challenge(req: Request) -> str:
return 'Basic realm="messenger", charset="UTF-8"' if req.url.path == "/api/v1/auth/sessions" else 'Bearer realm="messenger"'
def bearer_token(req: Request) -> str:
value = req.headers.get("Authorization", "")
if not value.startswith("Bearer ") or len(value) <= 7 or " " in value[7:] or len(value) > 512:
raise unauthorized()
try:
value[7:].encode("ascii")
except UnicodeEncodeError:
raise unauthorized() from None
return value[7:]
def session_key(token: str) -> str:
return "session:" + hashlib.sha256(token.encode("ascii")).hexdigest()
def current_user(req: Request) -> dict:
token = bearer_token(req)
try:
raw = cache().get(session_key(token))
except (RedisError, UnicodeError) as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
if not raw:
raise unauthorized()
try:
ident = str(uuid.UUID(raw))
except (ValueError, TypeError):
try:
cache().delete(session_key(token))
except RedisError as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
raise unauthorized() from None
with db() as conn:
row = conn.execute("SELECT * FROM users WHERE id = %s", (ident,)).fetchone()
if row is None:
raise unauthorized()
return public_user(row)
def login(req: Request):
auth = req.headers.get("Authorization", "")
if not auth.startswith("Basic "):
raise unauthorized()
try:
decoded = base64.b64decode(auth[6:].encode("ascii"), validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
if not PASSWORD_RE.fullmatch(username):
raise ValueError()
except (ValueError, UnicodeError, binascii.Error):
raise unauthorized() from None
with db() as conn:
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
if not row:
raise unauthorized()
try:
hasher.verify(row["password_hash"], password)
except (VerifyMismatchError, VerificationError):
raise unauthorized() from None
token = secrets.token_urlsafe(32)
try:
cache().set(session_key(token), str(row["id"]), ex=SESSION_TTL)
except RedisError as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
return {"token": token, "token_type": "Bearer", "expires_at": iso(datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)), "user": public_user(row)}
def logout(req: Request):
current_user(req)
try:
cache().delete(session_key(bearer_token(req)))
except RedisError as exc:
raise APIError(503, "unavailable", "Session store unavailable") from exc
+79
View File
@@ -0,0 +1,79 @@
"""Configuration, HTTP errors and shared representation helpers."""
import base64
import json
import os
import uuid
from datetime import datetime, timezone
from fastapi import Request
INSTANCE_ID = os.getenv("API_INSTANCE_ID", uuid.uuid4().hex[:12])
SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
IDEMPOTENCY_TTL = max(86400, int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")))
class APIError(Exception):
def __init__(self, status: int, code: str, message: str):
super().__init__(message)
self.status = status
self.code = code
self.message = message
def iso(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def invalid(message: str = "Invalid request") -> APIError:
return APIError(400, "invalid_request", message)
def parse_uuid(value: str) -> str:
try:
return str(uuid.UUID(value))
except (ValueError, TypeError, AttributeError):
raise invalid("Invalid UUID") from None
def parse_limit(raw: str | None) -> int:
if raw is None:
return 50
try:
if not raw.isascii() or not raw.isdecimal():
raise ValueError()
value = int(raw)
except (ValueError, AttributeError):
raise invalid("Invalid limit") from None
if not 1 <= value <= 100:
raise invalid("Invalid limit")
return value
def make_cursor(kind: str, scope: str, created: datetime, ident: str) -> str:
b = json.dumps({"kind": kind, "scope": scope, "created_at": iso(created), "id": ident}, separators=(",", ":")).encode()
return base64.urlsafe_b64encode(b).decode().rstrip("=")
def read_cursor(raw: str | None, kind: str, scope: str):
if raw is None:
return None
try:
if not raw or len(raw) > 2048 or any(ch not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" for ch in raw):
raise ValueError()
data = json.loads(base64.b64decode(raw + "=" * (-len(raw) % 4), altchars=b"-_", validate=True))
if not isinstance(data, dict) or data.get("kind") != kind or data.get("scope") != scope:
raise ValueError()
created = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
if created.tzinfo is None:
raise ValueError()
ident = str(uuid.UUID(data["id"]))
return created, ident
except (ValueError, TypeError, KeyError, UnicodeError, json.JSONDecodeError):
raise invalid("Invalid cursor") from None
def request_id(req: Request) -> str:
return getattr(req.state, "request_id", uuid.uuid4().hex)
def error_body(req: Request, code: str, message: str):
return {"error": {"code": code, "message": message, "request_id": request_id(req)}}
+124
View File
@@ -0,0 +1,124 @@
"""Sandboxed-per-task image work. Process is killed after JOB_TIMEOUT_SECONDS.
Outputs only JSON variant metadata. Original bytes are copied from durable staging,
never transformed. Pillow reopens/exif-transposes pixels for derivatives, stripping
metadata on re-encode. No API tokens or arbitrary user paths enter this command.
"""
import hashlib
import io
import json
import os
import sys
from PIL import Image, ImageOps, UnidentifiedImageError
from botocore.exceptions import ClientError, BotoCoreError
from .attachments import BUCKET, MAX_BYTES, MAX_PIXELS, s3_client
from . import telemetry as ot
from opentelemetry.trace import SpanKind
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
def rgb_image(src):
orient = ImageOps.exif_transpose(src)
orient.load()
if orient.width * orient.height > MAX_PIXELS:
raise ValueError("too many decoded pixels")
# Compose transparency on white, strip EXIF/ICC and do not enlarge.
rgba = orient.convert("RGBA")
white = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
white.alpha_composite(rgba)
return white.convert("RGB")
def transform(raw):
with Image.open(io.BytesIO(raw)) as image:
if image.format not in ("PNG", "JPEG"):
raise ValueError("invalid encoding")
rgb = rgb_image(image)
t = rgb.copy()
t.thumbnail((256, 256), resample=Image.Resampling.LANCZOS)
# Center crop fit scales up small images when needed.
crop = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
optim = rgb.copy()
optim.thumbnail((1280, 1280), resample=Image.Resampling.LANCZOS)
return [
("thumbnail", t, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}),
("crop", crop, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}),
("optimized", optim, "WEBP", "image/webp", {"quality": 78, "method": 4}),
]
def run(attachment_id, pipeline_version):
import psycopg
from psycopg.rows import dict_row
with ot.span("db.load_staging_metadata", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5) as conn:
row = conn.execute("""
SELECT * FROM attachments WHERE id=%s AND pipeline_version=%s
""", (attachment_id, pipeline_version)).fetchone()
if not row or not row["staging_key"]:
raise ValueError("staging metadata not found")
client = s3_client()
body = client.get_object(Bucket=BUCKET, Key=row["staging_key"])["Body"]
try:
raw = body.read(MAX_BYTES + 1)
finally:
body.close()
if not raw or len(raw) > MAX_BYTES or len(raw) != row["size_bytes"]:
raise ValueError("staging size mismatch")
if hashlib.sha256(raw).hexdigest() != row["sha256"]:
raise ValueError("staging checksum mismatch")
# Recompute every result if a prior run was interrupted. Keys are deterministic
# for (attachment_id,pipeline_version,variant), so duplicates don't multiply objects.
output = []
for name, image, fmt, mime, options in transform(raw):
buf = io.BytesIO()
image.save(buf, format=fmt, **options)
encoded = buf.getvalue()
key = f"variants/{attachment_id}/v{pipeline_version}/{name}"
client.put_object(Bucket=BUCKET, Key=key, Body=encoded, ContentType=mime)
output.append({"name": name, "content_type": mime,
"size_bytes": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(),
"width": image.width, "height": image.height, "object_key": key})
# Only after derivatives exist, publish unchanged primary and recovery copy.
client.copy_object(Bucket=BUCKET, Key=row["object_key"],
CopySource={"Bucket": BUCKET, "Key": row["staging_key"]},
MetadataDirective="REPLACE", ContentType=row["content_type"])
client.copy_object(Bucket=BUCKET, Key=row["backup_key"],
CopySource={"Bucket": BUCKET, "Key": row["object_key"]},
MetadataDirective="REPLACE", ContentType=row["content_type"])
return output
def main():
import uuid
if len(sys.argv) != 3:
raise SystemExit(2)
try:
ident = str(uuid.UUID(sys.argv[1]))
version = int(sys.argv[2])
if version < 1:
raise ValueError("invalid version")
ot.init("worker", child=True)
with ot.span("image.transform", context=ot.extract(json.loads(os.getenv("LAB6_PARENT_CONTEXT", "{}")))):
out = run(ident, version)
ot.event("image.transformed")
print(json.dumps({"variants": out}, ensure_ascii=False))
except (ValueError, UnidentifiedImageError, Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
print("Permanent image processing failure", file=sys.stderr)
raise SystemExit(20) from None
except (BotoCoreError, ClientError, OSError):
print("Transient object storage failure", file=sys.stderr)
raise SystemExit(30) from None
if __name__ == "__main__":
try:
main()
finally:
ot.flush()
+470
View File
@@ -0,0 +1,470 @@
"""Durable RabbitMQ delivery + PostgreSQL transactional outbox and worker leases.
There is no exactly-once transport guarantee. We guarantee eventual recovery of
committed intent and idempotent DB results under at-least-once redelivery.
Commands: `python -m app.jobs dispatch`, `worker`, `replay --id UUID`, `status`.
"""
import argparse
from contextlib import contextmanager
import json
import logging
import os
import random
import subprocess
import sys
import time
import threading
from . import telemetry as ot
from opentelemetry.trace import SpanKind, Status, StatusCode
import uuid
from datetime import datetime, timezone, timedelta
import pika
import psycopg
from pika.exceptions import AMQPError
from psycopg.rows import dict_row
log = logging.getLogger("lab5.jobs")
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
QUEUE = "attachment-processing.v1"
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
LEASE_SECONDS = JOB_TIMEOUT + 45
@contextmanager
def connect_db():
# No credentials, SQL text, or parameters in spans.
with ot.dependency_span('db.job.transaction', kind=SpanKind.CLIENT, attrs={'db.system.name':'postgresql'}):
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5, options='-c statement_timeout=5000') as conn:
yield conn
def job_payload(job_id, attachment_id, version, attempt):
return {
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
"attachment_id": str(attachment_id), "pipeline_version": version,
"attempt": attempt, "created_at": datetime.now(timezone.utc).isoformat(),
**ot.inject(),
}
@contextmanager
def connection():
params = pika.ConnectionParameters(
host=os.getenv("BROKER_HOST", "broker"), port=5672,
credentials=pika.PlainCredentials(os.environ["BROKER_USER"], os.environ["BROKER_PASSWORD"]),
heartbeat=max(120, JOB_TIMEOUT + 60), blocked_connection_timeout=120,
socket_timeout=5, connection_attempts=3, retry_delay=1)
mq = pika.BlockingConnection(params)
try:
yield mq
finally:
try:
if mq.is_open:
mq.close()
except AMQPError:
pass
def setup_channel(conn):
ch = conn.channel()
ch.queue_declare(queue=QUEUE, durable=True)
return ch
def queue_outbox(db, job_id, attachment_id, version, attempt, available_at=None):
payload = job_payload(job_id, attachment_id, version, attempt)
if not payload.get('traceparent'):
# Lease rescue/replay has no active consumer context. Preserve the first
# acceptance context rather than inventing a new disconnected trace.
previous = db.execute("SELECT payload FROM job_outbox WHERE job_id=%s ORDER BY created_at,id LIMIT 1", (job_id,)).fetchone()
if previous:
payload.update({k: previous['payload'][k] for k in ('traceparent','tracestate') if k in previous['payload']})
db.execute("""
INSERT INTO job_outbox(id,job_id,payload,available_at)
VALUES (%s,%s,%s::jsonb,COALESCE(%s,now()))
""", (str(uuid.uuid4()), job_id, json.dumps(payload), available_at))
def cleanup_abandoned_staging():
"""Best effort cleanup of retained stages; never delete current processing work."""
from .attachments import BUCKET, s3_client
client = s3_client()
retention = max(3600, int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
with connect_db() as db:
rows = db.execute("""
SELECT id,staging_key FROM attachments
WHERE staging_key IS NOT NULL AND
((status='ready' AND updated_at < now()-interval '300 seconds') OR
(status='failed' AND updated_at < now()-(%s*interval '1 second')))
LIMIT 50
""", (retention,)).fetchall()
for row in rows:
with connect_db() as db:
with db.transaction():
# Row-level lock serializes against operator replay.
current = db.execute("SELECT status,staging_key,updated_at FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)).fetchone()
if not current or current["staging_key"] != row["staging_key"]:
continue
age = (datetime.now(timezone.utc) - current["updated_at"]).total_seconds()
if not (current["status"] == "ready" and age >= 300 or current["status"] == "failed" and age >= retention):
continue
client.delete_object(Bucket=BUCKET, Key=row["staging_key"])
db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s", (row["id"], row["staging_key"]))
# Also prune staging uploads whose DB COMMIT failed, after a long retention.
from botocore.exceptions import ClientError
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET, Prefix="staging/"):
for obj in page.get("Contents", []):
if (datetime.now(timezone.utc) - obj["LastModified"]).total_seconds() < retention:
continue
key = obj["Key"]
with connect_db() as db:
with db.transaction():
referenced = db.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
if not referenced:
client.delete_object(Bucket=BUCKET, Key=key)
def rescue_expired_jobs(conn):
"""Requeue leased jobs whose worker died (durably, no manual task recreation)."""
with conn.transaction():
rows = conn.execute("""
SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt
FROM processing_jobs j
WHERE j.status='processing' AND j.leased_until < now()
FOR UPDATE SKIP LOCKED LIMIT 20
""").fetchall()
for row in rows:
if row["attempt"] >= MAX_ATTEMPTS:
conn.execute("""UPDATE processing_jobs
SET status='failed',error_code='retry_exhausted',lease_token=NULL,leased_until=NULL,updated_at=now()
WHERE id=%s""", (row["id"],))
conn.execute("""UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now()
WHERE id=%s""", (row["attachment_id"],))
else:
attempt = row["attempt"] + 1
conn.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,
lease_token=NULL,leased_until=NULL,next_run_at=now(),updated_at=now()
WHERE id=%s""", (attempt, row["id"]))
conn.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (row["attachment_id"],))
queue_outbox(conn, row["id"], row["attachment_id"], row["pipeline_version"], attempt)
log.warning("reclaimed timed-out job %s attempt %s", row["id"], attempt)
def sample_backlog():
# Query durable *queued* jobs, including delayed retries and unpublished outbox.
# No broker membership scan; oldest age is wall clock since initial acceptance.
while True:
try:
with psycopg.connect(os.environ['DATABASE_URL'], row_factory=dict_row, connect_timeout=3, options='-c statement_timeout=2000') as db:
row = db.execute("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS age FROM processing_jobs WHERE status='queued'").fetchone()
ot.backlog_depth, ot.backlog_age = row['depth'], float(row['age'])
except psycopg.Error:
ot.backlog_depth = ot.backlog_age = None
time.sleep(3)
def dispatch_loop():
"""Publish outbox rows with publisher confirm, mark only AFTER confirm."""
threading.Thread(target=sample_backlog, daemon=True).start()
while True:
try:
with connection() as mq:
ch = setup_channel(mq)
ch.confirm_delivery()
last_clean = time.monotonic()
last_queue_sample = 0
while True:
# Periodic lease recovery works even if original message was consumed.
with connect_db() as conn:
rescue_expired_jobs(conn)
with conn.transaction():
rows = conn.execute("""
SELECT id,payload FROM job_outbox
WHERE published_at IS NULL AND available_at<=now()
ORDER BY available_at,created_at
FOR UPDATE SKIP LOCKED LIMIT 20
""").fetchall()
for row in rows:
if os.getenv("LAB5_FAULT_BEFORE_PUBLISH") == "once":
# Persist one-time fault before hard exit; outbox still
# unmarked and dispatcher will publish on restart.
with connect_db() as faultdb:
inserted = faultdb.execute("""
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'before_publish')
ON CONFLICT DO NOTHING RETURNING job_id
""", (row["payload"]["job_id"],)).fetchone()
if inserted:
os._exit(78)
payload = row['payload']
with ot.span('rabbitmq.publish', kind=SpanKind.PRODUCER, context=ot.extract(payload),
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE,'job.attempt':payload['attempt']}) as producer:
# Propagate actual producer span, while DB retains the original intent.
raw = json.dumps({**payload, **ot.inject()}, separators=(",", ":"))
ch.basic_publish(exchange="", routing_key=QUEUE, body=raw.encode("utf-8"),
mandatory=True, properties=pika.BasicProperties(content_type="application/json", delivery_mode=2))
conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=%s", (row["id"],))
ot.event('job.published', job_id=payload['job_id'], attempt=payload['attempt'])
if time.monotonic() - last_queue_sample > 3:
q = ch.queue_declare(queue=QUEUE, passive=True).method
ot.broker_ready, ot.broker_consumers = q.message_count, q.consumer_count
last_queue_sample = time.monotonic()
if time.monotonic() - last_clean > 60:
try:
cleanup_abandoned_staging()
except Exception:
log.warning("staging janitor failed; retry later")
last_clean = time.monotonic()
if not rows:
mq.sleep(1)
except (AMQPError, OSError, psycopg.Error) as exc:
ot.broker_ready = ot.broker_consumers = None
log.warning("dispatcher reconnect after %s", type(exc).__name__)
time.sleep(2)
def claim_job(db, message):
job_id = message["job_id"]
with db.transaction():
job = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job_id,)).fetchone()
if job is None or job["status"] in ("ready", "failed") or job["attempt"] != message["attempt"] or job["pipeline_version"] != message["pipeline_version"]:
return "ignore", None
if job["status"] == "processing":
return "busy", None
if job["next_run_at"] > datetime.now(timezone.utc):
return "busy", None
lease = str(uuid.uuid4())
db.execute("""UPDATE processing_jobs SET status='processing',lease_token=%s,
leased_until=now()+(%s*interval '1 second'),updated_at=now()
WHERE id=%s""", (lease, LEASE_SECONDS, job_id))
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (job["attachment_id"],))
return "claimed", {"job": job, "lease": lease}
def publish_result(db, *, job, lease, variants):
"""Final result is committed atomically; stale leases/versions cannot publish."""
with db.transaction():
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
if current["status"] != "processing" or str(current["lease_token"]) != lease or current["pipeline_version"] != job["pipeline_version"]:
log.warning("stale worker result ignored %s", job["id"])
return False
for v in variants:
db.execute("""
INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
ON CONFLICT(attachment_id,pipeline_version,name) DO UPDATE SET
object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type,
size_bytes=EXCLUDED.size_bytes,sha256=EXCLUDED.sha256,
width=EXCLUDED.width,height=EXCLUDED.height
""", (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"],
v["content_type"], v["size_bytes"], v["sha256"], v["width"], v["height"]))
db.execute("""UPDATE attachments SET status='ready',error_code=NULL,updated_at=now()
WHERE id=%s AND pipeline_version=%s""", (job["attachment_id"], job["pipeline_version"]))
db.execute("""UPDATE processing_jobs SET status='ready',lease_token=NULL,leased_until=NULL,
error_code=NULL,updated_at=now() WHERE id=%s""", (job["id"],))
return True
def fail_job(db, job, lease, *, permanent=False, code="processing_failed"):
with db.transaction():
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
if current["status"] != "processing" or str(current["lease_token"]) != lease:
return "stale"
if permanent or job["attempt"] >= MAX_ATTEMPTS:
code = code if permanent else "retry_exhausted"
db.execute("""UPDATE processing_jobs SET status='failed',error_code=%s,lease_token=NULL,
leased_until=NULL,updated_at=now() WHERE id=%s""", (code, job["id"]))
db.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s", (code, job["attachment_id"]))
return "failed"
else:
attempt = job["attempt"] + 1
delay = min(60.0, 2.0 * (2 ** (attempt - 2))) + random.uniform(0, 1.0)
run_at = datetime.now(timezone.utc) + timedelta(seconds=delay)
db.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,next_run_at=%s,
lease_token=NULL,leased_until=NULL,updated_at=now() WHERE id=%s""", (attempt, run_at, job["id"]))
db.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (job["attachment_id"],))
queue_outbox(db, job["id"], job["attachment_id"], job["pipeline_version"], attempt, run_at)
log.warning("job %s scheduled retry %s after %.1fs", job["id"], attempt, delay)
return "retry"
def cleanup_staging(job):
from .attachments import BUCKET, s3_client
from botocore.exceptions import BotoCoreError, ClientError
try:
with connect_db() as conn:
row = conn.execute("SELECT staging_key,status FROM attachments WHERE id=%s", (job["attachment_id"],)).fetchone()
if row and row["status"] == "ready" and row["staging_key"]:
s3_client().delete_object(Bucket=BUCKET, Key=row["staging_key"])
with connect_db() as conn:
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND status='ready' AND staging_key=%s",
(job["attachment_id"], row["staging_key"]))
except (BotoCoreError, ClientError, psycopg.Error, OSError):
log.warning("staging cleanup deferred for %s", job["id"])
def process_message(raw):
try:
carrier = json.loads(raw)
if not isinstance(carrier, dict): carrier = {}
except (ValueError, TypeError):
carrier = {}
with ot.span('rabbitmq.consume', kind=SpanKind.CONSUMER, context=ot.extract(carrier),
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE}) as consumer:
return process_message_inner(raw, consumer)
def process_message_inner(raw, consumer):
try:
msg = json.loads(raw)
assert msg["schema_version"] == 1 and msg["type"] == "attachment.process.v1"
uuid.UUID(msg["job_id"])
uuid.UUID(msg["attachment_id"])
assert type(msg["attempt"]) is int and msg["attempt"] >= 1
except (ValueError, KeyError, AssertionError, TypeError):
log.error("invalid internal broker message discarded")
return "ack" # no secrets printed, does not touch DB
with connect_db() as db:
status, state = claim_job(db, msg)
if status == "ignore":
return "ack"
if status == "busy":
return "requeue"
job, lease = state["job"], state["lease"]
consumer.set_attribute('job.attempt',job['attempt'])
consumer.set_attribute('job.id',str(job['id']))
ot.event('job.claimed', job_id=str(job['id']), attempt=job['attempt'])
start = time.monotonic()
ot.worker_busy = 1
def done(outcome):
attrs = {'job_type':'attachment.process.v1','outcome':outcome}
ot.count('lab_jobs',attrs)
ot.observe('lab_job_duration',time.monotonic()-start,attrs)
if outcome == 'retry':
ot.count('lab_job_retries',{'job_type':'attachment.process.v1'})
if outcome in ('retry','failed','requeue'):
consumer.set_status(Status(StatusCode.ERROR))
consumer.set_attribute('job.outcome',outcome)
ot.event('job.'+outcome, level=logging.ERROR if outcome in ('failed','retry','requeue') else logging.INFO, job_id=str(job['id']), attempt=job['attempt'])
try:
if os.getenv('LAB6_FAULT_RETRY_ONCE') == 'once':
with connect_db() as db:
inserted = db.execute("INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'lab6_retry') ON CONFLICT DO NOTHING RETURNING job_id", (job['id'],)).fetchone()
if inserted:
raise RuntimeError('injected transient retry')
cmd = [sys.executable, "-m", "app.image_worker", str(job["attachment_id"]), str(job["pipeline_version"])]
with ot.span('image.subprocess'):
env = {**os.environ, 'LAB6_PARENT_CONTEXT': json.dumps(ot.inject())}
proc = subprocess.run(cmd, capture_output=True, timeout=JOB_TIMEOUT, check=False, env=env)
if proc.returncode != 0:
with connect_db() as db:
outcome = fail_job(db, job, lease, permanent=(proc.returncode == 20),
code="invalid_image" if proc.returncode == 20 else "processing_failed")
done(outcome)
return "ack" # next attempt is already safely in DB outbox
result = json.loads(proc.stdout)
variants = result["variants"]
if sorted(v["name"] for v in variants) != ["crop", "optimized", "thumbnail"]:
raise ValueError("incomplete variants")
if os.getenv("LAB5_FAULT_AFTER_S3") == "once":
# One-time operator-only injected hard death, after all S3 writes and
# before DB result commit/ack; SQL PK prevents endless failure loop.
with connect_db() as db:
with db.transaction():
inserted = db.execute("""
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'after_s3')
ON CONFLICT DO NOTHING RETURNING job_id
""", (job["id"],)).fetchone()
if inserted:
os._exit(77)
with ot.span("db.publish_result", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
with connect_db() as db:
ok = publish_result(db, job=job, lease=lease, variants=variants)
if ok:
ot.observe('lab_ready_duration',(datetime.now(timezone.utc)-job['created_at']).total_seconds(), {'job_type':'attachment.process.v1'})
cleanup_staging(job)
done('ready' if ok else 'stale')
return "ack"
except subprocess.TimeoutExpired:
with connect_db() as db:
outcome = fail_job(db, job, lease, code="processing_timeout")
done(outcome)
return "ack"
except Exception:
log.exception("job processing failure (internal details not sent to API)")
# Leave lease for dispatcher recovery if even DB unavailable.
try:
with connect_db() as db:
outcome = fail_job(db, job, lease)
done(outcome)
return "ack"
except psycopg.Error:
done("requeue")
return "requeue"
finally:
ot.worker_busy = 0
def worker_loop():
while True:
try:
with connection() as mq:
ch = setup_channel(mq)
ch.basic_qos(prefetch_count=max(1, int(os.getenv("WORKER_PREFETCH", "1"))))
def callback(channel, method, props, body):
outcome = process_message(body)
if outcome == "ack":
channel.basic_ack(method.delivery_tag)
else:
channel.basic_nack(method.delivery_tag, requeue=True)
time.sleep(2) # prevent tight redelivery spin on an active lease
ch.basic_consume(queue=QUEUE, on_message_callback=callback, auto_ack=False)
log.info("worker ready; max attempts=%d timeout=%ds", MAX_ATTEMPTS, JOB_TIMEOUT)
ch.start_consuming()
except (AMQPError, OSError, psycopg.Error) as exc:
log.warning("worker reconnect after %s", type(exc).__name__)
time.sleep(2)
def replay_failed(ident):
with connect_db() as db:
with db.transaction():
row = db.execute("SELECT * FROM processing_jobs WHERE attachment_id=%s FOR UPDATE", (ident,)).fetchone()
if not row or row["status"] != "failed":
raise RuntimeError("Only failed jobs may be explicitly replayed")
attachment = db.execute("SELECT staging_key FROM attachments WHERE id=%s", (ident,)).fetchone()
if not attachment or not attachment["staging_key"]:
raise RuntimeError("Retained staging is required for replay")
db.execute("""UPDATE processing_jobs SET status='queued',attempt=1,next_run_at=now(),
lease_token=NULL,leased_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s""", (row["id"],))
db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s", (ident,))
queue_outbox(db, row["id"], ident, row["pipeline_version"], 1)
db.execute("INSERT INTO job_replay_audit(id,job_id) VALUES (%s,%s)", (str(uuid.uuid4()), row["id"]))
print("Replayed failed job", ident)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("mode", choices=["dispatch", "worker", "replay", "status"])
parser.add_argument("--id", help="UUID for explicit operator replay/status")
args = parser.parse_args()
if args.mode == "dispatch":
ot.init("dispatcher")
dispatch_loop()
elif args.mode == "worker":
ot.init("worker")
worker_loop()
elif args.mode in ("replay", "status"):
if not args.id:
parser.error("--id is required")
ident = str(uuid.UUID(args.id))
if args.mode == "replay":
replay_failed(ident)
else:
with connect_db() as db:
print(json.dumps(db.execute("SELECT status,attempt,error_code FROM processing_jobs WHERE attachment_id=%s", (ident,)).fetchone(), default=str))
if __name__ == "__main__":
main()
+117
View File
@@ -0,0 +1,117 @@
"""FastAPI process: one stateless replica. No auto-migration on startup."""
import logging
import uuid
import time
from starlette.routing import Match
from opentelemetry.trace import SpanKind, Status, StatusCode
from . import telemetry as ot
from contextlib import asynccontextmanager
import psycopg
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.concurrency import run_in_threadpool
from redis.exceptions import RedisError
from .api import router
from .attachments import router as attachment_router
from .auth import challenge, current_user
from .core import APIError, INSTANCE_ID, error_body, invalid, request_id
from .storage import start, stop
logging.basicConfig(level="INFO")
logger = logging.getLogger("lab5")
@asynccontextmanager
async def lifespan(app: FastAPI):
ot.init("api")
start()
try:
yield
finally:
stop()
ot.flush()
app = FastAPI(title="MTUCI Messenger Lab 6", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan)
app.include_router(router)
app.include_router(attachment_router)
PROTECTED = ("/api/v1/chats", "/api/v1/attachments", "/api/v1/users/me", "/api/v1/auth/sessions/current")
JSON_POST = ("/api/v1/users", "/api/v1/chats")
@app.middleware("http")
async def request_context(req: Request, call_next):
# Match the registered template even when early authentication fails.
route = next((r.path for r in app.routes if r.matches(req.scope)[0] == Match.FULL), 'unmatched')
method = req.method if req.method in ('GET','POST','DELETE','PUT','PATCH','HEAD','OPTIONS') else 'OTHER'
attrs = {'method': method, 'route': route}
start_at = time.monotonic()
if route in ("/health/live", "/health/ready"):
response = await handle_request(req, call_next)
attrs["status_class"] = str(response.status_code//100)+"xx"
ot.count("lab_http_requests", attrs)
ot.observe("lab_http_duration", time.monotonic()-start_at, attrs)
return response
with ot.span(method+' '+route, kind=SpanKind.SERVER, attrs={'http.request.method':method,'http.route':route}) as span:
response = await handle_request(req, call_next)
attrs['status_class'] = str(response.status_code//100)+'xx'
span.set_attribute('http.response.status_code', response.status_code)
if response.status_code >= 500:
span.set_status(Status(StatusCode.ERROR))
ot.count('lab_http_requests', attrs)
ot.observe('lab_http_duration', time.monotonic()-start_at, attrs)
ot.event('http.completed', level=logging.ERROR if response.status_code >= 500 else logging.INFO,
request_id=request_id(req), route=route, method=method, status_code=response.status_code)
if ot.trace_id():
response.headers['X-Trace-Id'] = ot.trace_id()
return response
async def handle_request(req: Request, call_next):
req.state.request_id = uuid.uuid4().hex
try:
path = req.url.path
# Authenticate before parsing the request body or resource-specific fields.
if any(path == p or path.startswith(p + "/") for p in PROTECTED):
req.state.actor = await run_in_threadpool(current_user, req)
if req.method == "POST" and (path in JSON_POST or (path.startswith("/api/v1/chats/") and path.endswith("/messages"))):
if req.headers.get("content-type", "").split(";", 1)[0].strip().lower() != "application/json":
raise APIError(415, "unsupported_media_type", "Content-Type must be application/json")
if req.method == "POST" and path == "/api/v1/auth/sessions" and await req.body():
raise invalid("Login body is not allowed")
response = await call_next(req)
except APIError as exc:
response = api_error(req, exc)
except (RedisError, psycopg.OperationalError, psycopg.InterfaceError):
response = api_error(req, APIError(503, "unavailable", "Dependency unavailable"))
except Exception:
logger.exception("unexpected request failure id=%s", request_id(req))
response = api_error(req, APIError(500, "internal_error", "Internal server error"))
response.headers["X-Request-Id"] = request_id(req)
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
def api_error(req: Request, exc: APIError) -> JSONResponse:
headers = {"WWW-Authenticate": challenge(req)} if exc.status == 401 else {}
return JSONResponse(status_code=exc.status, content=error_body(req, exc.code, exc.message), headers=headers)
@app.exception_handler(APIError)
async def api_error_handler(req: Request, exc: APIError):
return api_error(req, exc)
@app.exception_handler(RequestValidationError)
async def validation_error_handler(req: Request, exc: RequestValidationError):
return api_error(req, invalid())
@app.exception_handler(Exception)
async def unexpected_error_handler(req: Request, exc: Exception):
logger.exception("handler failure id=%s", request_id(req))
if isinstance(exc, (psycopg.OperationalError, psycopg.InterfaceError, RedisError)):
return api_error(req, APIError(503, "unavailable", "Dependency unavailable"))
return api_error(req, APIError(500, "internal_error", "Internal server error"))
+23
View File
@@ -0,0 +1,23 @@
"""Run once before API instances; transactional and advisory-lock coordinated migrations."""
import os
from pathlib import Path
import psycopg
MIGRATIONS = Path(__file__).resolve().parents[1] / "db" / "migrations"
def main():
with psycopg.connect(os.environ["DATABASE_URL"], autocommit=False) as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(62132103)")
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())")
for path in sorted(MIGRATIONS.glob("[0-9]*.sql")):
applied = conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone()
if applied:
continue
conn.execute(path.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (path.name,))
print("Applied", path.name, flush=True)
print("Migrations up to date", flush=True)
if __name__ == "__main__":
main()
+77
View File
@@ -0,0 +1,77 @@
"""HTTP request models; all strings use Unicode code point limits."""
import re
import uuid
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
class CreateUser(StrictModel):
username: str
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_valid(cls, value: str) -> str:
if not USERNAME_RE.fullmatch(value):
raise ValueError("Invalid username")
return value
@field_validator("display_name", "password")
@classmethod
def non_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("Blank value")
value.encode("utf-8", errors="strict")
return value
class CreateChat(StrictModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def non_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("Blank title")
value.encode("utf-8", errors="strict")
return value
@field_validator("member_ids")
@classmethod
def uuid_members(cls, value: list[str]) -> list[str]:
for member in value:
if not isinstance(member, str) or str(uuid.UUID(member)) != member:
raise ValueError("Invalid member UUID")
if len(value) != len(set(value)):
raise ValueError("Duplicate member")
return value
class CreateMessage(StrictModel):
text: str = Field(min_length=0, max_length=4000)
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
@field_validator("text")
@classmethod
def non_blank(cls, value: str) -> str:
value.encode("utf-8", errors="strict")
return value
@field_validator("attachment_ids")
@classmethod
def valid_attachments(cls, value: list[str]) -> list[str]:
if len(value) != len(set(value)):
raise ValueError("Duplicate attachment IDs")
for item in value:
if not isinstance(item, str) or str(uuid.UUID(item)) != item:
raise ValueError("Invalid attachment UUID")
return value
@model_validator(mode="after")
def nonempty_message(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Empty message")
return self
+25
View File
@@ -0,0 +1,25 @@
"""Offline image conversion tests inside the application image; no S3 or DB."""
import io
from PIL import Image
from .image_worker import transform
def main():
source = Image.new('RGB',(320,200), (255,0,0))
buf=io.BytesIO();source.save(buf,'PNG')
result=transform(buf.getvalue())
items={name:(img,fmt,mime,opts) for name,img,fmt,mime,opts in result}
assert set(items)=={'thumbnail','crop','optimized'}
assert items['thumbnail'][0].size==(256,160)
assert items['crop'][0].size==(128,128)
assert items['optimized'][0].size==(320,200)
for name,(im,fmt,mime,opts) in items.items():
data=io.BytesIO();im.save(data,format=fmt,**opts)
with Image.open(io.BytesIO(data.getvalue())) as decoded:
assert decoded.size==im.size
assert decoded.format==fmt
assert 'exif' not in decoded.info
print('PASS: PNG 320x200 -> JPEG thumb 256x160, crop 128x128, optimized WebP, metadata stripped')
if __name__=='__main__':
main()
+132
View File
@@ -0,0 +1,132 @@
"""Lab4 grade-5 track 2: repeatable DB/S3 reconciliation.
Dry-run is default. Apply never drops metadata when BOTH object copies are gone:
without a trustworthy copy or external backup, deletion would destroy evidence.
"""
import argparse
import json
import sys
import time
from datetime import datetime, timezone
import psycopg
from psycopg.rows import dict_row
from botocore.exceptions import BotoCoreError, ClientError
from .attachments import BUCKET, s3_client
def inventory(client):
result = {}
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET):
for obj in page.get("Contents", []):
# Only reconciliation-owned prefixes; never mutate arbitrary S3 keys.
key = obj["Key"]
if key.startswith(("attachments/", "recovery/")):
result[key] = obj["LastModified"]
return result
def get_db_rows(conn):
return conn.execute("SELECT id, object_key, backup_key, status FROM attachments").fetchall()
def current_reference(conn, key):
return conn.execute(
"SELECT 1 FROM attachments WHERE object_key=%s OR backup_key=%s", (key, key)
).fetchone() is not None
def reconcile(*, apply=False, grace_seconds=300, now=None, client=None, connect=None):
if grace_seconds < 300 and apply:
raise ValueError("Apply requires >=300 s grace period to protect in-flight uploads")
started = time.monotonic()
now = now or datetime.now(timezone.utc)
client = client or s3_client()
connect = connect or (lambda: psycopg.connect(__import__('os').environ['DATABASE_URL'], row_factory=dict_row))
objects = inventory(client)
stats = {
"mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds,
"s3_keys_scanned": len(objects), "db_rows_scanned": 0,
"orphan_keys": [], "orphan_keys_young": [],
"missing_primary_repairable": [], "missing_backup_repairable": [],
"missing_both_unrecoverable": [], "actions": [], "errors": [],
}
with connect() as conn:
rows = get_db_rows(conn)
stats["db_rows_scanned"] = len(rows)
references = {key for row in rows for key in (row["object_key"], row["backup_key"])}
for row in rows:
if row["status"] != "ready":
continue # queued/processing have durable staging, not final variants yet
primary, backup = row["object_key"], row["backup_key"]
have_primary, have_backup = primary in objects, backup in objects
rid = str(row["id"])
if have_primary and have_backup:
continue
if not have_primary and not have_backup:
stats["missing_both_unrecoverable"].append(rid)
continue
dest, source, typ = (primary, backup, "missing_primary_repairable") if not have_primary else (backup, primary, "missing_backup_repairable")
stats[typ].append(rid)
if apply:
# Defensive recheck. Object may have reappeared after inventory.
try:
client.head_object(Bucket=BUCKET, Key=dest)
stats["actions"].append({"action": "already-restored", "key": dest})
continue
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code")
if code not in ("404", "NotFound", "NoSuchKey"):
raise
try:
client.copy_object(Bucket=BUCKET, Key=dest,
CopySource={"Bucket": BUCKET, "Key": source})
stats["actions"].append({"action": "restore", "key": dest, "from": source})
except (BotoCoreError, ClientError) as exc:
stats["errors"].append({"action": "restore", "key": dest, "error_type": type(exc).__name__})
for key, modified in objects.items():
if key in references:
continue
age = (now - modified).total_seconds()
if age < grace_seconds:
stats["orphan_keys_young"].append(key)
continue
stats["orphan_keys"].append(key)
if apply:
# Query again immediately before destructive action; never delete
# an object now referenced by a committed metadata row.
if current_reference(conn, key):
stats["actions"].append({"action": "skip-now-referenced", "key": key})
continue
try:
client.delete_object(Bucket=BUCKET, Key=key)
stats["actions"].append({"action": "delete-old-orphan", "key": key})
except (BotoCoreError, ClientError) as exc:
stats["errors"].append({"action": "delete-old-orphan", "key": key, "error_type": type(exc).__name__})
stats["scan_elapsed_seconds"] = round(time.monotonic() - started, 3)
return stats
def main():
parser = argparse.ArgumentParser(description="Reconcile S3 attachments vs PostgreSQL (dry-run default)")
parser.add_argument("--apply", action="store_true", help="Make safe changes; orphans older than 300 seconds only")
parser.add_argument("--grace-seconds", type=int, default=300)
parser.add_argument("--output", help="Write JSON summary (no credentials)")
args = parser.parse_args()
try:
result = reconcile(apply=args.apply, grace_seconds=args.grace_seconds)
except Exception as exc:
# No exception string: botocore errors may contain signed credentials/URLs.
print(json.dumps({"status":"failed", "error_type":type(exc).__name__}), file=sys.stderr)
raise SystemExit(2) from None
rendered = json.dumps(result, ensure_ascii=False, indent=2)
print(rendered)
if args.output:
with open(args.output, "w", encoding="utf-8") as fh:
fh.write(rendered + "\n")
if result["errors"] or result["missing_both_unrecoverable"]:
raise SystemExit(1)
if __name__ == "__main__":
main()
+36
View File
@@ -0,0 +1,36 @@
"""Operator-only fault injection in a LOCAL TEST stack; never exposed as HTTP."""
import argparse
import json
import os
import uuid
import psycopg
from psycopg.rows import dict_row
from .attachments import BUCKET, s3_client
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["delete-primary", "delete-backup", "create-orphan"])
p.add_argument("--id", help="Attachment UUID for the delete actions")
args = p.parse_args()
if os.environ.get("LAB4_ALLOW_FAULT_INJECTION") != "yes":
raise SystemExit("Fault injection disabled; use LAB4_ALLOW_FAULT_INJECTION=yes in isolated test")
client = s3_client()
if args.action == "create-orphan":
key = f"attachments/{uuid.uuid4()}/original"
client.put_object(Bucket=BUCKET, Key=key, Body=b"synthetic orphan")
print(json.dumps({"fault":"orphan-created", "object_key":key}))
return
if not args.id:
raise SystemExit("--id is required")
ident = str(uuid.UUID(args.id))
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row) as conn:
row = conn.execute("SELECT object_key,backup_key FROM attachments WHERE id=%s", (ident,)).fetchone()
if not row:
raise SystemExit("Attachment does not exist")
key = row["object_key"] if args.action == "delete-primary" else row["backup_key"]
client.delete_object(Bucket=BUCKET, Key=key)
print(json.dumps({"fault":args.action, "object_key":key}))
if __name__ == "__main__":
main()
+61
View File
@@ -0,0 +1,61 @@
"""Fast isolated tests for reconciliation safety; run in application image."""
import uuid
from datetime import datetime, timezone, timedelta
from .reconcile import reconcile
from .attachments import BUCKET
NOW = datetime.now(timezone.utc)
class FakeConn:
def __init__(self, rows): self.rows=rows
def __enter__(self): return self
def __exit__(self,*a): return False
def execute(self, sql, args=None):
self.sql=sql
self.args=args
return self
def fetchall(self): return self.rows
def fetchone(self):
key=self.args[0]
return {'id': 1} if any(key in (row['object_key'],row['backup_key']) for row in self.rows) else None
class FakeS3:
def __init__(self,objects): self.objects=objects
def get_paginator(self,op): return self
def paginate(self,**kwargs):
yield {'Contents':[{'Key':k,'LastModified':t} for k,t in self.objects.items()]}
def head_object(self,**kw):
if kw['Key'] not in self.objects:
from botocore.exceptions import ClientError
raise ClientError({'Error':{'Code':'404'}},'HeadObject')
def copy_object(self,**kw):
assert kw['CopySource']['Key'] in self.objects
self.objects[kw['Key']]=NOW
def delete_object(self,**kw):
self.objects.pop(kw['Key'],None)
def test():
i=str(uuid.uuid4())
primary=f'attachments/{i}/original';backup=f'recovery/{i}/original'
young=f'attachments/{uuid.uuid4()}/original'
old=f'attachments/{uuid.uuid4()}/original'
rows=[{'id':i,'object_key':primary,'backup_key':backup,'status':'ready'}]
fake=FakeS3({backup: NOW-timedelta(days=1),young:NOW-timedelta(seconds=10),old:NOW-timedelta(days=1)})
kwargs={'now':NOW,'client':fake,'connect':lambda:FakeConn(rows)}
first=reconcile(apply=False,**kwargs)
assert first['missing_primary_repairable']==[i]
assert old in first['orphan_keys'] and young in first['orphan_keys_young']
assert primary not in fake.objects and old in fake.objects, 'Dry run mutated objects'
second=reconcile(apply=True,**kwargs)
assert primary in fake.objects and old not in fake.objects and young in fake.objects
assert any(x['action']=='restore' for x in second['actions'])
third=reconcile(apply=True,**kwargs)
assert not third['actions'], 'Re-run unexpectedly mutated objects'
assert rows[0]['object_key']==primary, 'Metadata destroyed'
try: reconcile(apply=True, grace_seconds=0, **kwargs)
except ValueError: pass
else: raise AssertionError('Unsafe zero-grace apply accepted')
print('PASS: dry-run, orphan grace, recover primary from backup, apply idempotence')
if __name__ == '__main__': test()
+35
View File
@@ -0,0 +1,35 @@
"""Idempotently create the private S3 bucket with bootstrap-only credentials."""
import os
import time
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError, BotoCoreError
def main():
bucket = os.environ["S3_BUCKET"]
client = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ADMIN_ACCESS_KEY"],
aws_secret_access_key=os.environ["S3_ADMIN_SECRET_KEY"],
region_name="us-east-1",
config=Config(s3={"addressing_style": "path"},
connect_timeout=2, read_timeout=4, retries={"max_attempts": 1}))
for attempt in range(70):
try:
try:
client.head_bucket(Bucket=bucket)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code", "")
if code not in ("404", "NoSuchBucket", "NotFound"):
raise
client.create_bucket(Bucket=bucket)
# PRIVATE: no public bucket policy, ACL or anonymous identity.
print(f"Private bucket {bucket} ready", flush=True)
return
except (ClientError, BotoCoreError, OSError) as exc:
if attempt == 69:
raise RuntimeError("S3 bootstrap failed; inspect S3 container") from exc
time.sleep(1)
if __name__ == "__main__":
main()
+32
View File
@@ -0,0 +1,32 @@
"""One-off creation of private SeaweedFS S3 identities; secrets stay in Docker volume."""
import json
import os
from pathlib import Path
def main():
bucket = os.environ["S3_BUCKET"]
identities = [
{"name": "bucket-bootstrap", "credentials": [{"accessKey": os.environ["S3_ADMIN_ACCESS_KEY"],
"secretKey": os.environ["S3_ADMIN_SECRET_KEY"]}],
"actions": ["Admin", "Read", "Write", "List"]},
{"name": "messenger-api", "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY"],
"secretKey": os.environ["S3_SECRET_KEY"]}],
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]},
]
if len({entry["credentials"][0]["accessKey"] for entry in identities}) != 2:
raise ValueError("Admin and API credentials must differ")
path = Path("/s3-config/config.json")
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(".tmp")
old_umask = os.umask(0o077)
try:
tmp.write_text(json.dumps({"identities": identities}, separators=(",", ":")), encoding="utf-8")
os.chmod(tmp, 0o644) # readable by the SeaweedFS runtime UID; Docker volume stays private
os.replace(tmp, path)
finally:
os.umask(old_umask)
print("S3 identities configured (no secrets printed)", flush=True)
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
"""Verify bucket is private at S3 boundary (not merely hidden behind HTTP API)."""
import json
import os
import boto3
from botocore import UNSIGNED
from botocore.config import Config
from botocore.exceptions import ClientError
from .attachments import BUCKET, s3_client
def main():
client = s3_client()
client.head_bucket(Bucket=BUCKET) # restricted app identity works
unauth = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"],
region_name="us-east-1", config=Config(signature_version=UNSIGNED,
s3={"addressing_style":"path"}))
try:
unauth.list_objects_v2(Bucket=BUCKET, MaxKeys=1)
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status not in (401,403):
raise RuntimeError("Unexpected anonymous S3 response") from None
else:
raise RuntimeError("Private bucket leaked object list to anonymous caller")
listing = client.list_objects_v2(Bucket=BUCKET, MaxKeys=1)
keys = [entry["Key"] for entry in listing.get("Contents", [])]
if keys:
try:
unauth.get_object(Bucket=BUCKET, Key=keys[0])
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status not in (401, 403):
raise RuntimeError("Unexpected anonymous object GET response") from None
else:
raise RuntimeError("Private S3 object downloadable anonymously")
print(json.dumps({"private_bucket": True, "anonymous_listing": "denied",
"anonymous_object_get": "denied" if keys else "not_tested_empty_bucket"}))
if __name__ == "__main__":
main()
+67
View File
@@ -0,0 +1,67 @@
"""Shared external state: psycopg connection pool and Redis session store."""
import os
from contextlib import contextmanager
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
from redis import Redis
from redis.exceptions import RedisError
from .core import APIError
from . import telemetry as ot
from opentelemetry.trace import SpanKind
pool: ConnectionPool | None = None
redis_client: Redis | None = None
def start() -> None:
global pool, redis_client
url = os.environ["DATABASE_URL"]
size = int(os.getenv("DB_POOL_MAX", "8"))
pool = ConnectionPool(conninfo=url, min_size=1, max_size=size, timeout=3, kwargs={"row_factory": dict_row, "connect_timeout": 5, "options": "-c statement_timeout=5000"}, open=False)
pool.open(wait=True, timeout=30)
redis_client = Redis.from_url(os.environ.get("REDIS_URL", "redis://redis:6379/0"), socket_connect_timeout=2, socket_timeout=2, decode_responses=True)
redis_client.ping()
def stop() -> None:
global pool, redis_client
if pool is not None:
pool.close()
pool = None
if redis_client is not None:
redis_client.close()
redis_client = None
@contextmanager
def db():
if pool is None:
raise APIError(503, "unavailable", "Database unavailable")
try:
with ot.dependency_span("db.transaction", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
with pool.connection() as conn:
yield conn
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled, PoolTimeout) as exc:
raise APIError(503, "unavailable", "Database unavailable") from exc
def cache() -> Redis:
if redis_client is None:
raise APIError(503, "unavailable", "Session store unavailable")
return redis_client
def ready() -> bool:
try:
with db() as conn:
conn.execute("SELECT 1").fetchone()
cache().ping()
# S3 is needed for the mandatory synchronous upload/download path.
from .attachments import s3_client, BUCKET
s3_client().head_bucket(Bucket=BUCKET)
return True
except Exception:
return False
+204
View File
@@ -0,0 +1,204 @@
"""Explicit, allowlisted OTLP instrumentation; no automatic headers, SQL or bodies.
SDK exporters run asynchronously with bounded queues; telemetry is best effort.
"""
import logging
import os
import socket
import time
from contextlib import contextmanager, nullcontext
from pathlib import Path
from opentelemetry import metrics, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
PROP = TraceContextTextMapPropagator()
tracer = trace.get_tracer('messenger.lab6')
log = logging.getLogger('messenger.telemetry')
log.setLevel(logging.INFO)
log.propagate = False
providers = []
started = False
m = None
instruments = {}
worker_busy = 0
backlog_depth = None
backlog_age = None
broker_ready = None
broker_consumers = None
QUEUE = 'attachment-processing.v1'
BUCKETS = (0.005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 15, 30, 60, 120, 300, 600)
def init(role, *, child=False):
global started, tracer, m
if started or os.getenv('OTEL_ENABLED', 'no') != 'yes':
return
started = True
resource = Resource.create({'service.name': 'messenger-' + role,
'service.version': '1.6.0', 'service.instance.id': os.getenv('API_INSTANCE_ID', socket.gethostname()),
'deployment.environment.name': 'lab'})
base = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
tp = TracerProvider(resource=resource)
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=base+'/v1/traces', timeout=2),
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
trace.set_tracer_provider(tp)
tracer = tp.get_tracer('messenger.lab6')
lp = LoggerProvider(resource=resource)
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=base+'/v1/logs', timeout=2),
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
log.addHandler(LoggingHandler(logger_provider=lp))
providers.extend([tp, lp])
if child:
return
reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=base+'/v1/metrics', timeout=2),
export_interval_millis=3000, export_timeout_millis=2000)
mp = MeterProvider(resource=resource, metric_readers=[reader], views=[View(instrument_name=n,
aggregation=ExplicitBucketHistogramAggregation(boundaries=BUCKETS)) for n in
('lab_http_duration','lab_job_duration','lab_ready_duration')])
metrics.set_meter_provider(mp)
providers.append(mp)
m = mp.get_meter('messenger.lab6')
for name in ('lab_http_requests','lab_jobs','lab_job_retries'):
instruments[name] = m.create_counter(name)
for name in ('lab_http_duration','lab_job_duration','lab_ready_duration'):
instruments[name] = m.create_histogram(name, unit='s')
if role == 'api':
for name, key in [('lab_db_pool_used','used'),('lab_db_pool_waiting','requests_waiting'),('lab_db_pool_capacity','pool_max')]:
m.create_observable_gauge(name, callbacks=[lambda _, key=key: pool_stats(key)])
if role == 'worker':
m.create_observable_gauge('lab_worker_busy', callbacks=[lambda _: [metrics.Observation(worker_busy)]])
m.create_observable_gauge('lab_worker_capacity', callbacks=[lambda _: [metrics.Observation(1)]])
if role == 'dispatcher':
m.create_observable_gauge('lab_broker_ready', callbacks=[lambda _: queue_observation(broker_ready)])
m.create_observable_gauge('lab_broker_consumers', callbacks=[lambda _: queue_observation(broker_consumers)])
m.create_observable_gauge('lab_queue_depth', callbacks=[lambda _: queue_observation(backlog_depth)])
m.create_observable_gauge('lab_queue_oldest_age', unit='s', callbacks=[lambda _: queue_observation(backlog_age)])
m.create_observable_gauge('lab_container_memory', unit='By', callbacks=[lambda _: container_observation('memory')])
m.create_observable_gauge('lab_container_cpu_cores', callbacks=[lambda _: container_observation('cpu')])
def pool_stats(key):
from . import storage
if storage.pool is None:
return []
stats = storage.pool.get_stats()
val = stats.get('pool_size',0)-stats.get('pool_available',0) if key=='used' else stats.get(key,0)
return [metrics.Observation(max(0,val))]
def queue_observation(value):
return [] if value is None else [metrics.Observation(value, {'queue': QUEUE})]
_cpu_previous = None
def container_observation(kind):
global _cpu_previous
try:
if kind == 'memory':
return [metrics.Observation(int(Path('/sys/fs/cgroup/memory.current').read_text()))]
cpu = int(dict(line.split() for line in Path('/sys/fs/cgroup/cpu.stat').read_text().splitlines())['usage_usec']) / 1e6
now = time.monotonic()
old = _cpu_previous
_cpu_previous = (now,cpu)
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
except (OSError, KeyError, ValueError):
# cgroup v1 fallback, process tree (not full container accounting).
import psutil
try:
p = psutil.Process()
processes = [p]+p.children(recursive=True)
if kind == 'memory':
value = 0
for proc in processes:
try: value += proc.memory_info().rss
except psutil.Error: pass
return [metrics.Observation(value)]
# cgroup v1 (Docker Linux) if available, CPU usage in nanoseconds.
for path in ('/sys/fs/cgroup/cpuacct/cpuacct.usage','/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage'):
if Path(path).is_file():
cpu = int(Path(path).read_text()) / 1e9
now = time.monotonic()
old = _cpu_previous
_cpu_previous = (now,cpu)
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
except (psutil.Error,OSError,ValueError):
pass
return []
def count(name, attrs=None):
if name in instruments: instruments[name].add(1, attrs or {})
def observe(name, value, attrs=None):
if name in instruments: instruments[name].record(max(0,value), attrs or {})
def event(name, *, level=logging.INFO, **attrs):
# Callers pass fixed event names and technical ID/status only, never payloads.
if started:
log.log(level, name, extra={'event':name, **attrs})
@contextmanager
def span(name, *, kind=SpanKind.INTERNAL, context=None, attrs=None):
with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attrs,
record_exception=False, set_status_on_exception=False) as s:
try:
yield s
except BaseException as exc:
s.set_status(Status(StatusCode.ERROR))
s.set_attribute('error.type', type(exc).__name__)
event('operation.error', level=logging.ERROR, operation=name, error_type=type(exc).__name__)
raise
def inject():
result = {}
PROP.inject(result)
return result
def extract(carrier):
# Ignore baggage and all unrelated fields.
return PROP.extract({k:v for k,v in carrier.items() if k in ('traceparent','tracestate') and isinstance(v,str)})
def dependency_span(name, **kwargs):
return span(name, **kwargs) if trace_id() else nullcontext()
def trace_id():
ctx = trace.get_current_span().get_span_context()
return f'{ctx.trace_id:032x}' if ctx.is_valid else ''
def flush():
for p in providers:
p.force_flush(timeout_millis=2500)
class S3Tracing:
"""Only operation name is exported; bucket/key/bytes/signed URLs never exported."""
def __init__(self, client): self.client = client
def __getattr__(self, name):
f = getattr(self.client,name)
if name not in ('upload_fileobj','get_object','put_object','copy_object','delete_object','head_object','head_bucket'):
return f
def call(*a, **kw):
with dependency_span('s3.'+name, kind=SpanKind.CLIENT, attrs={'storage.system':'s3'}):
return f(*a, **kw)
return call
+347
View File
@@ -0,0 +1,347 @@
services:
postgres:
image: postgres:16.10-bookworm
environment:
POSTGRES_DB: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-lab}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
volumes: ["postgres_data:/var/lib/postgresql/data"]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: 1.0
restart: unless-stopped
networks: [storage]
redis:
image: redis:7.4.5-alpine
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", "60", "1"]
volumes: ["redis_data:/data"]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 256m
cpus: 0.5
restart: unless-stopped
networks: [storage]
broker:
image: rabbitmq:4.1.8-management-alpine
hostname: lab6-broker
environment:
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-labworker}
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Set BROKER_PASSWORD in .env}
volumes: ["rabbitmq_data:/var/lib/rabbitmq"]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 25
start_period: 15s
mem_limit: 512m
cpus: 0.75
restart: unless-stopped
networks: [storage]
s3-policy:
build: .
user: "0:0"
command: ["python", "-m", "app.s3_policy"]
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:?Set S3_ADMIN_ACCESS_KEY}
S3_ADMIN_SECRET_KEY: ${S3_ADMIN_SECRET_KEY:?Set S3_ADMIN_SECRET_KEY}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
volumes: ["s3_config:/s3-config"]
restart: "no"
networks: [storage]
s3:
image: chrislusf/seaweedfs:4.53
command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.port=8333", "-s3.iam=false", "-s3.config=/s3-config/config.json"]
volumes:
- s3_data:/data
- s3_config:/s3-config:ro
- ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro
depends_on:
s3-policy:
condition: service_completed_successfully
expose: ["8333"]
mem_limit: 640m
cpus: 1.0
restart: unless-stopped
networks: [storage]
s3-bootstrap:
build: .
command: ["python", "-m", "app.s3_bootstrap"]
environment:
S3_ENDPOINT: http://s3:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:?Set S3_ADMIN_ACCESS_KEY}
S3_ADMIN_SECRET_KEY: ${S3_ADMIN_SECRET_KEY:?Set S3_ADMIN_SECRET_KEY}
depends_on:
s3:
condition: service_started
restart: "no"
networks: [storage]
migrate:
build: .
command: ["python", "-m", "app.migrate"]
environment:
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
depends_on:
postgres:
condition: service_healthy
restart: "no"
networks: [storage]
reconcile:
profiles: [maintenance]
build: .
command: ["python", "-m", "app.reconcile"]
environment:
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
S3_ENDPOINT: http://s3:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
depends_on:
postgres:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
restart: "no"
networks: [storage]
dispatcher:
build: .
command: ["python", "-m", "app.jobs", "dispatch"]
environment: &job_env
OTEL_ENABLED: ${OTEL_ENABLED:-yes}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
BROKER_HOST: broker
BROKER_USER: ${BROKER_USER:-labworker}
BROKER_PASSWORD: ${BROKER_PASSWORD:?Set BROKER_PASSWORD in .env}
S3_ENDPOINT: http://s3:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
depends_on:
broker:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
mem_limit: 192m
cpus: 0.5
restart: unless-stopped
networks: [storage, telemetry]
worker:
build: .
command: ["python", "-m", "app.jobs", "worker"]
environment:
<<: *job_env
LAB6_FAULT_RETRY_ONCE: ${LAB6_FAULT_RETRY_ONCE:-off}
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
LAB5_FAULT_AFTER_S3: ${LAB5_FAULT_AFTER_S3:-off}
depends_on:
broker:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
mem_limit: 768m
cpus: 1.0
stop_grace_period: 75s
restart: unless-stopped
networks: [storage, telemetry]
api1:
build: .
environment: &api_env
OTEL_ENABLED: ${OTEL_ENABLED:-yes}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
APP_PORT: "8080"
API_INSTANCE_ID: api1
DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DB_POOL_MAX: ${DB_POOL_MAX:-8}
JOB_BACKLOG_LIMIT: ${JOB_BACKLOG_LIMIT:-1000}
S3_ENDPOINT: http://s3:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
depends_on: &api_depends
postgres:
condition: service_healthy
redis:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
expose: ["8080"]
healthcheck: &api_health
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 2s
timeout: 3s
retries: 15
start_period: 10s
mem_limit: 384m
cpus: 0.75
stop_grace_period: 12s
restart: unless-stopped
networks: [frontend, storage, telemetry]
api2:
build: .
environment:
<<: *api_env
API_INSTANCE_ID: api2
depends_on: *api_depends
expose: ["8080"]
healthcheck: *api_health
mem_limit: 384m
cpus: 0.75
stop_grace_period: 12s
restart: unless-stopped
networks: [frontend, storage, telemetry]
lb:
image: haproxy:3.1.7-alpine
volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"]
ports: ["127.0.0.1:8080:8080"]
depends_on:
api1:
condition: service_healthy
api2:
condition: service_healthy
healthcheck:
test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
interval: 10s
timeout: 3s
retries: 3
mem_limit: 128m
cpus: 0.5
restart: unless-stopped
networks: [frontend]
telemetry-init:
image: busybox:1.37.0
user: "0:0"
command: ["sh", "-ec", "chown 10001:10001 /tempo /loki; chown 65534:65534 /prometheus; chown 472:472 /grafana"]
volumes:
- tempo_data:/tempo
- loki_data:/loki
- prometheus_data:/prometheus
- grafana_data:/grafana
network_mode: none
restart: "no"
otel-collector:
image: otel/opentelemetry-collector-contrib:0.123.0
command: ["--config=/etc/otelcol/config.yaml"]
volumes: ["./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro"]
environment:
GOMEMLIMIT: 180MiB
mem_limit: 256m
cpus: 0.75
restart: unless-stopped
networks: [telemetry]
prometheus:
depends_on:
telemetry-init:
condition: service_completed_successfully
image: prom/prometheus:v3.2.1
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=72h", "--storage.tsdb.retention.size=512MB"]
volumes:
- ./infra/otel/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/otel/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
- prometheus_data:/prometheus
ports: ["127.0.0.1:9090:9090"]
mem_limit: 384m
restart: unless-stopped
networks: [telemetry]
loki:
depends_on:
telemetry-init:
condition: service_completed_successfully
image: grafana/loki:3.4.2
command: ["-config.file=/etc/loki/local-config.yaml"]
volumes:
- ./infra/otel/loki.yaml:/etc/loki/local-config.yaml:ro
- loki_data:/loki
ports: ["127.0.0.1:3100:3100"]
mem_limit: 512m
restart: unless-stopped
networks: [telemetry]
tempo:
depends_on:
telemetry-init:
condition: service_completed_successfully
image: grafana/tempo:2.7.2
command: ["-config.file=/etc/tempo.yaml"]
volumes:
- ./infra/otel/tempo.yaml:/etc/tempo.yaml:ro
- tempo_data:/var/tempo
ports: ["127.0.0.1:3200:3200"]
mem_limit: 512m
restart: unless-stopped
networks: [telemetry]
grafana:
depends_on:
telemetry-init:
condition: service_completed_successfully
image: grafana/grafana:11.6.0
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env}
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
volumes:
- grafana_data:/var/lib/grafana
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
ports: ["127.0.0.1:3000:3000"]
mem_limit: 384m
restart: unless-stopped
networks: [telemetry]
networks:
telemetry:
frontend:
storage:
internal: true
volumes:
postgres_data:
redis_data:
s3_data:
s3_config:
rabbitmq_data:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
+33
View File
@@ -0,0 +1,33 @@
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username VARCHAR(32) NOT NULL UNIQUE,
display_name BYTEA NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_display_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400)
);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title BYTEA NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400)
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id),
user_id UUID NOT NULL REFERENCES users(id),
position INTEGER NOT NULL,
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX IF NOT EXISTS idx_members_user ON chat_members (user_id, chat_id);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id),
sender_id UUID NOT NULL,
text BYTEA NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id),
CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000)
);
CREATE INDEX IF NOT EXISTS idx_messages_chat_history ON messages (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_chats_history ON chats (created_at, id);
+12
View File
@@ -0,0 +1,12 @@
-- Same transaction creates message and idempotency receipt. Unique key arbitrates concurrent replicas.
CREATE TABLE IF NOT EXISTS message_idempotency (
actor_id UUID NOT NULL REFERENCES users(id),
chat_id UUID NOT NULL REFERENCES chats(id),
method VARCHAR(8) NOT NULL DEFAULT 'POST',
idem_key VARCHAR(128) NOT NULL,
body_sha256 CHAR(64) NOT NULL,
message_id UUID NOT NULL REFERENCES messages(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (actor_id, chat_id, method, idem_key)
);
CREATE INDEX IF NOT EXISTS idx_idempotency_age ON message_idempotency (created_at);
+25
View File
@@ -0,0 +1,25 @@
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_bytes;
ALTER TABLE messages ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000);
CREATE TABLE IF NOT EXISTS attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id),
owner_id UUID NOT NULL REFERENCES users(id),
object_key TEXT NOT NULL UNIQUE,
backup_key TEXT NOT NULL UNIQUE,
filename TEXT NOT NULL,
content_type VARCHAR(32) NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
size_bytes BIGINT NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
sha256 CHAR(64) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id);
CREATE TABLE IF NOT EXISTS message_attachments (
message_id UUID NOT NULL REFERENCES messages(id),
attachment_id UUID NOT NULL REFERENCES attachments(id),
position INTEGER NOT NULL CHECK(position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, attachment_id),
UNIQUE (message_id,position)
);
CREATE INDEX IF NOT EXISTS idx_message_attachments_attachment ON message_attachments(attachment_id);
+58
View File
@@ -0,0 +1,58 @@
-- Lab5: stage durably, then transact job and outbox atomically.
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'ready'
CHECK (status IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code TEXT;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INTEGER NOT NULL DEFAULT 1;
CREATE UNIQUE INDEX IF NOT EXISTS idx_attachments_staging ON attachments(staging_key) WHERE staging_key IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(status);
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id UUID NOT NULL REFERENCES attachments(id),
pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1),
name TEXT NOT NULL CHECK (name IN ('thumbnail','crop','optimized')),
object_key TEXT NOT NULL UNIQUE,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL CHECK (size_bytes > 0),
sha256 CHAR(64) NOT NULL,
width INTEGER NOT NULL CHECK (width > 0),
height INTEGER NOT NULL CHECK (height > 0),
PRIMARY KEY (attachment_id,pipeline_version,name)
);
CREATE TABLE IF NOT EXISTS processing_jobs (
id UUID PRIMARY KEY,
attachment_id UUID NOT NULL UNIQUE REFERENCES attachments(id),
pipeline_version INTEGER NOT NULL DEFAULT 1,
status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued','processing','ready','failed')),
attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1),
next_run_at TIMESTAMPTZ NOT NULL DEFAULT now(),
lease_token UUID,
leased_until TIMESTAMPTZ,
error_code TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_jobs_leases ON processing_jobs(status, leased_until);
CREATE TABLE IF NOT EXISTS job_outbox (
id UUID PRIMARY KEY,
job_id UUID NOT NULL REFERENCES processing_jobs(id),
payload JSONB NOT NULL,
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
published_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_outbox_due ON job_outbox(available_at, created_at) WHERE published_at IS NULL;
CREATE TABLE IF NOT EXISTS job_replay_audit (
id UUID PRIMARY KEY,
job_id UUID NOT NULL REFERENCES processing_jobs(id),
replayed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
action TEXT NOT NULL DEFAULT 'operator_replay'
);
CREATE TABLE IF NOT EXISTS job_fault_once (
job_id UUID NOT NULL REFERENCES processing_jobs(id),
fault TEXT NOT NULL,
PRIMARY KEY (job_id,fault)
);
+13
View File
@@ -0,0 +1,13 @@
{
"concurrency": 10,
"accepted_uploads": 10,
"ready_attachments": 10,
"failed_requests": 0,
"acceptance_elapsed_seconds": 2.515,
"acceptance_rps": 3.976,
"upload_latency_p50_ms": 2296.5,
"upload_latency_max_ms": 2484.0,
"all_ready_elapsed_seconds": 36.406,
"bytes_per_file": 111817,
"note": "HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats."
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 113 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 79 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 150 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 111 KiB

+16
View File
@@ -0,0 +1,16 @@
{
"scenario": "backend-down",
"status": "PASS",
"result": {
"successful_messages_during_tempo_outage": 16,
"trace_id": "9aa3b3a670850e443df5779f0f60d8a8",
"max_sampled_collector_rss_bytes": 210169856.0,
"collector_memory_limit_bytes": 268435456,
"rss_sample_count": 17,
"max_export_queue": 18.0,
"export_queue_capacity": 64.0,
"failed_span_counter_delta": 21.0,
"restart_state": "0 false",
"recovered_trace_id": "89192e038ce76d9f10bc2d714af7f77d"
}
}
+30
View File
@@ -0,0 +1,30 @@
{
"scenario": "db-error",
"status": "PASS",
"result": {
"trace_id": "859975662df88e09af421eb97ce16911",
"request_id": "174f09c4345a425d91854d9d614fc1ca",
"http_codes": [
503,
503,
503,
503,
503,
503,
503,
503,
503,
503,
503,
503
],
"firing_error_ratio": 1.0,
"resolved_error_ratio": 0.0,
"error_spans": [
{
"name": "db.transaction",
"error_type": "QueryCanceled"
}
]
}
}
+10
View File
@@ -0,0 +1,10 @@
{
"scenario": "history",
"status": "PASS",
"result": {
"trace_id": "24926a7aa71429d9947f0c17f08b63a2",
"historical_query_timestamp": 1794929612.5405939,
"metric_before": 148.0,
"metric_after": 148.0
}
}
+532
View File
@@ -0,0 +1,532 @@
{
"scenario": "metrics",
"status": "PASS",
"result": {
"api_rps": {
"query": "sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s]))",
"result": [
{
"metric": {},
"value": [
1794929001.859,
"0.4074074074074075"
]
}
]
},
"error_ratio": {
"query": "(sum(rate(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s])),0.001)",
"result": [
{
"metric": {},
"value": [
1794929001.864,
"0"
]
}
]
},
"p50": {
"query": "histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])))",
"result": [
{
"metric": {},
"value": [
1794929001.88,
"0.017503056800918893"
]
}
]
},
"p95": {
"query": "histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])))",
"result": [
{
"metric": {},
"value": [
1794929001.891,
"0.0725"
]
}
]
},
"p99": {
"query": "histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])))",
"result": [
{
"metric": {},
"value": [
1794929001.897,
"0.09450000000000001"
]
}
]
},
"jobs": {
"query": "lab_jobs_total",
"result": [
{
"metric": {
"__name__": "lab_jobs_total",
"deployment_environment_name": "lab",
"exported_instance": "9d2e6ea768ae",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"job_type": "attachment.process.v1",
"outcome": "ready",
"service_instance_id": "9d2e6ea768ae",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929001.904,
"2"
]
}
]
},
"job_histogram": {
"query": "lab_job_duration_seconds_count",
"result": [
{
"metric": {
"__name__": "lab_job_duration_seconds_count",
"deployment_environment_name": "lab",
"exported_instance": "9d2e6ea768ae",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"job_type": "attachment.process.v1",
"outcome": "ready",
"service_instance_id": "9d2e6ea768ae",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929001.927,
"2"
]
}
]
},
"ready_histogram": {
"query": "lab_ready_duration_seconds_count",
"result": [
{
"metric": {
"__name__": "lab_ready_duration_seconds_count",
"deployment_environment_name": "lab",
"exported_instance": "9d2e6ea768ae",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"job_type": "attachment.process.v1",
"service_instance_id": "9d2e6ea768ae",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929001.932,
"2"
]
}
]
},
"depth": {
"query": "max(lab_queue_depth)",
"result": [
{
"metric": {},
"value": [
1794929001.937,
"0"
]
}
]
},
"oldest": {
"query": "max(lab_queue_oldest_age_seconds)",
"result": [
{
"metric": {},
"value": [
1794929001.957,
"0"
]
}
]
},
"broker_ready": {
"query": "lab_broker_ready",
"result": [
{
"metric": {
"__name__": "lab_broker_ready",
"deployment_environment_name": "lab",
"exported_instance": "23fa9474ac6a",
"exported_job": "messenger-dispatcher",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"queue": "attachment-processing.v1",
"service_instance_id": "23fa9474ac6a",
"service_name": "messenger-dispatcher",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929001.973,
"0"
]
}
]
},
"broker_consumers": {
"query": "lab_broker_consumers",
"result": [
{
"metric": {
"__name__": "lab_broker_consumers",
"deployment_environment_name": "lab",
"exported_instance": "23fa9474ac6a",
"exported_job": "messenger-dispatcher",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"queue": "attachment-processing.v1",
"service_instance_id": "23fa9474ac6a",
"service_name": "messenger-dispatcher",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929001.988,
"1"
]
}
]
},
"pool_used": {
"query": "lab_db_pool_used",
"result": [
{
"metric": {
"__name__": "lab_db_pool_used",
"deployment_environment_name": "lab",
"exported_instance": "api1",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "api1",
"service_name": "messenger-api",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929001.993,
"0"
]
},
{
"metric": {
"__name__": "lab_db_pool_used",
"deployment_environment_name": "lab",
"exported_instance": "api2",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "api2",
"service_name": "messenger-api",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929001.993,
"0"
]
}
]
},
"pool_waiting": {
"query": "lab_db_pool_waiting",
"result": [
{
"metric": {
"__name__": "lab_db_pool_waiting",
"deployment_environment_name": "lab",
"exported_instance": "api1",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "api1",
"service_name": "messenger-api",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.019,
"0"
]
},
{
"metric": {
"__name__": "lab_db_pool_waiting",
"deployment_environment_name": "lab",
"exported_instance": "api2",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "api2",
"service_name": "messenger-api",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.019,
"0"
]
}
]
},
"capacity": {
"query": "lab_worker_capacity",
"result": [
{
"metric": {
"__name__": "lab_worker_capacity",
"deployment_environment_name": "lab",
"exported_instance": "9d2e6ea768ae",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "9d2e6ea768ae",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.036,
"1"
]
}
]
},
"busy": {
"query": "lab_worker_busy",
"result": [
{
"metric": {
"__name__": "lab_worker_busy",
"deployment_environment_name": "lab",
"exported_instance": "9d2e6ea768ae",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "9d2e6ea768ae",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.067,
"0"
]
}
]
},
"cpu": {
"query": "lab_container_cpu_cores",
"result": [
{
"metric": {
"__name__": "lab_container_cpu_cores",
"deployment_environment_name": "lab",
"exported_instance": "9d2e6ea768ae",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "9d2e6ea768ae",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.098,
"0.0012147006450733576"
]
},
{
"metric": {
"__name__": "lab_container_cpu_cores",
"deployment_environment_name": "lab",
"exported_instance": "api1",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "api1",
"service_name": "messenger-api",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.098,
"0.1248036553953329"
]
},
{
"metric": {
"__name__": "lab_container_cpu_cores",
"deployment_environment_name": "lab",
"exported_instance": "api2",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "api2",
"service_name": "messenger-api",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.098,
"0.1264468449809642"
]
},
{
"metric": {
"__name__": "lab_container_cpu_cores",
"deployment_environment_name": "lab",
"exported_instance": "23fa9474ac6a",
"exported_job": "messenger-dispatcher",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "23fa9474ac6a",
"service_name": "messenger-dispatcher",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.098,
"0.009300433365947897"
]
}
]
},
"memory": {
"query": "lab_container_memory_bytes",
"result": [
{
"metric": {
"__name__": "lab_container_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "9d2e6ea768ae",
"exported_job": "messenger-worker",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "9d2e6ea768ae",
"service_name": "messenger-worker",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.107,
"104075264"
]
},
{
"metric": {
"__name__": "lab_container_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "api1",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "api1",
"service_name": "messenger-api",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.107,
"152047616"
]
},
{
"metric": {
"__name__": "lab_container_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "api2",
"exported_job": "messenger-api",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "api2",
"service_name": "messenger-api",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.107,
"144863232"
]
},
{
"metric": {
"__name__": "lab_container_memory_bytes",
"deployment_environment_name": "lab",
"exported_instance": "23fa9474ac6a",
"exported_job": "messenger-dispatcher",
"instance": "otel-collector:8889",
"job": "lab-otlp",
"service_instance_id": "23fa9474ac6a",
"service_name": "messenger-dispatcher",
"service_version": "1.6.0",
"telemetry_sdk_language": "python",
"telemetry_sdk_name": "opentelemetry",
"telemetry_sdk_version": "1.42.1"
},
"value": [
1794929002.107,
"104079360"
]
}
]
}
}
}
+11
View File
@@ -0,0 +1,11 @@
{
"scenario": "queue-lag",
"status": "PASS",
"result": {
"trace_id": "6bf0ddf7b17bb3e846aa931aab6e97ef",
"firing_lag_seconds": 27.574387,
"firing_depth": 1.0,
"detected_seconds": 34.235,
"resolved_depth": 0.0
}
}
+214
View File
@@ -0,0 +1,214 @@
{
"scenario": "trace-retry",
"status": "PASS",
"result": {
"trace_id": "4d8711b71965876a59aa5594ad0d51a0",
"request_id": "d92704875e2b4f0cb3b76f5354943a99",
"spans": [
{
"name": "POST /api/v1/chats/{chat_id}/attachments",
"id": "k4uo/LzSEq8=",
"parent": null,
"attempt": null,
"outcome": null
},
{
"name": "db.transaction",
"id": "DzqQmZ0ugps=",
"parent": "k4uo/LzSEq8=",
"attempt": null,
"outcome": null
},
{
"name": "db.transaction",
"id": "2ChmWodQe24=",
"parent": "k4uo/LzSEq8=",
"attempt": null,
"outcome": null
},
{
"name": "s3.upload_fileobj",
"id": "Vy3PofMYPBU=",
"parent": "k4uo/LzSEq8=",
"attempt": null,
"outcome": null
},
{
"name": "db.transaction",
"id": "BmV9vHb3CTw=",
"parent": "k4uo/LzSEq8=",
"attempt": null,
"outcome": null
},
{
"name": "rabbitmq.publish",
"id": "eRBOkrMeoa8=",
"parent": "BmV9vHb3CTw=",
"attempt": "1",
"outcome": null
},
{
"name": "rabbitmq.consume",
"id": "8iVPBhXuImY=",
"parent": "eRBOkrMeoa8=",
"attempt": "1",
"outcome": "retry"
},
{
"name": "db.job.transaction",
"id": "lZp3w62oH14=",
"parent": "8iVPBhXuImY=",
"attempt": null,
"outcome": null
},
{
"name": "db.job.transaction",
"id": "TO5Q0Nf/ECg=",
"parent": "8iVPBhXuImY=",
"attempt": null,
"outcome": null
},
{
"name": "db.job.transaction",
"id": "aWYPqHrrvag=",
"parent": "8iVPBhXuImY=",
"attempt": null,
"outcome": null
},
{
"name": "rabbitmq.publish",
"id": "WBujc7E21qk=",
"parent": "aWYPqHrrvag=",
"attempt": "2",
"outcome": null
},
{
"name": "rabbitmq.consume",
"id": "jvmmPa040qY=",
"parent": "WBujc7E21qk=",
"attempt": "2",
"outcome": "ready"
},
{
"name": "db.job.transaction",
"id": "gRcYatAJcDI=",
"parent": "jvmmPa040qY=",
"attempt": null,
"outcome": null
},
{
"name": "s3.delete_object",
"id": "5apnWPa0LHs=",
"parent": "jvmmPa040qY=",
"attempt": null,
"outcome": null
},
{
"name": "db.job.transaction",
"id": "GHNw+OD6g+E=",
"parent": "jvmmPa040qY=",
"attempt": null,
"outcome": null
},
{
"name": "db.job.transaction",
"id": "t6ytfExz/wY=",
"parent": "jvmmPa040qY=",
"attempt": null,
"outcome": null
},
{
"name": "db.job.transaction",
"id": "nGilf7vU83w=",
"parent": "jvmmPa040qY=",
"attempt": null,
"outcome": null
},
{
"name": "image.subprocess",
"id": "30CroshuABo=",
"parent": "jvmmPa040qY=",
"attempt": null,
"outcome": null
},
{
"name": "db.publish_result",
"id": "FhqMuTZIy5E=",
"parent": "jvmmPa040qY=",
"attempt": null,
"outcome": null
},
{
"name": "db.job.transaction",
"id": "8it2Knjruyc=",
"parent": "FhqMuTZIy5E=",
"attempt": null,
"outcome": null
},
{
"name": "image.transform",
"id": "NmBuL1p0oMs=",
"parent": "30CroshuABo=",
"attempt": null,
"outcome": null
},
{
"name": "db.load_staging_metadata",
"id": "sDN1N6qO6A8=",
"parent": "NmBuL1p0oMs=",
"attempt": null,
"outcome": null
},
{
"name": "s3.get_object",
"id": "mf9t+iylMEA=",
"parent": "NmBuL1p0oMs=",
"attempt": null,
"outcome": null
},
{
"name": "s3.put_object",
"id": "AOPN3aX8c/k=",
"parent": "NmBuL1p0oMs=",
"attempt": null,
"outcome": null
},
{
"name": "s3.put_object",
"id": "yjZiBqAz1v0=",
"parent": "NmBuL1p0oMs=",
"attempt": null,
"outcome": null
},
{
"name": "s3.put_object",
"id": "9kNHAG8kgW4=",
"parent": "NmBuL1p0oMs=",
"attempt": null,
"outcome": null
},
{
"name": "s3.copy_object",
"id": "hRIkPPRqeho=",
"parent": "NmBuL1p0oMs=",
"attempt": null,
"outcome": null
},
{
"name": "s3.copy_object",
"id": "zhrE/ErGjRE=",
"parent": "NmBuL1p0oMs=",
"attempt": null,
"outcome": null
}
],
"worker_log_streams": 5,
"worker_events": [
"image.transformed",
"job.claimed",
"job.ready",
"job.retry"
],
"retry_chain": true
}
}
+18
View File
@@ -0,0 +1,18 @@
# Локальные результаты подготовки Lab6 (2026-11-17)
Не являются результатами Docker-интеграции. Среда подготовки: Linux/Python 3.13, установлен SDK OpenTelemetry 1.42.1, Docker CLI отсутствует. Файлы исходного задания проверены byte-for-byte: изменены только .env.example, Makefile (python вместо python3 для Windows) и REPORT.md. Исходные публичные проверки, CI, job schema, OpenAPI, README и PR template сохранены.
```text
make check
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
python scripts/check_lab6.py
PASS: 65 literal SQL bindings; pinned Compose images, mounts and networks; 3 OTLP pipelines, dashboards and alert rules (static only)
python tests/jobs_sql_contract.py
PASS: attachment_variants INSERT has 9 columns, 9 placeholders, 9 arguments
python tests/own_unit.py
PASS: models, cursor scoping, limits, Unicode U+0000
python tests/telemetry_unit.py
PASS: real OTLP trace/log/metric encoding, parent links, histogram buckets, safe S3 attributes
```
`telemetry_unit.py` использует настоящий Python SDK и локальный HTTP OTLP receiver, декодирует protobuf и проверяет parent IDs, trace/span IDs в логе и histogram boundary values; не заменяет проверку Collector, Loki, Tempo, Prometheus, Grafana. Дополнительно: все Python modules успешно разобраны AST; JSON Schema examples и job с traceparent прошли jsonschema validation. Full OpenAPI validator отсутствует в среде подготовки; полный `make validate` не исполнялся. Точная конфигурация/реальные команды backend проверяются на стенде при запуске. Бизнес-сценарии, backend failure, alert firing/resolved, telemetry persistence и нагрузка на Lab6 ещё не подтверждены.
+90
View File
@@ -0,0 +1,90 @@
# Подтверждённые результаты Lab6 — 17.11.2026
Целевая оценка: 4. Стенд: Windows, PowerShell, Docker Desktop (Linux containers). Источник — фактический вывод команд и четыре оригинальных PNG из Grafana, предоставленные после запуска Lab6. Это сводка наблюдений, не повторный прогон тестов. Версия commit и параметры оборудования не зафиксированы. Секреты не включены.
## Запуск и проверка конфигурации
| Команда | Результат |
| --- | --- |
| `docker compose --env-file .env config --quiet` | PASS, ошибки отсутствовали |
| `docker compose --env-file .env run --rm --no-deps otel-collector validate --config=/etc/otelcol/config.yaml` | PASS, ошибок в выводе нет |
| `docker compose --env-file .env run --rm --no-deps --entrypoint /bin/promtool prometheus check config /etc/prometheus/prometheus.yml` | SUCCESS: конфигурация валидна, 2 правила алертов |
| `docker compose --env-file .env up -d --build --wait` и `ps` | Сборка завершена; 14 основных контейнеров Up; API1/API2, broker, lb, PostgreSQL, Redis healthy |
| `make check` | PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. |
| `make test` | PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. |
## Наблюдаемость: реальные сценарии
Перед fault-тестами: `$env:LAB6_ALLOW_FAILURE_TEST = "yes"`.
| Команда (`python tests/observability_live.py --scenario ...`) | Наблюдаемый результат | Файл на стенде |
| --- | --- | --- |
| `metrics` | PASS: все ожидаемые ряды существуют, запрещённых ID labels нет | `lab6-metrics.json` |
| `trace-retry` | PASS: API/producer/consumer/S3/DB, retry и связанные логи | `lab6-trace-retry.json` |
| `queue-lag` | PASS: alert firing, oldest age 27.574387 s, queued=1. После восстановления worker: ready и resolved | `lab6-queue-lag.json` |
| `db-error` | PASS: DB lock → QueryCanceled/503, error ratio 1.0, alert firing → resolved | `lab6-db-error.json` |
| `backend-down` | PASS после исправления диагностического теста: 16 записей и обработка изображения во время отказа Tempo; ошибки экспорта видимы; RSS/очередь ограничены; свежая трасса восстановилась | `lab6-backend-down.json` |
| `history` | PASS: ранее сохранённые метрики, логи, трассы доступны после перезапуска backend | `lab6-history.json` |
Сквозная трасса: `4d8711b71965876a59aa5594ad0d51a0`. Может быть удалена политикой retention; при необходимости повторить `trace-retry`.
Первый запуск `backend-down` дал `FAIL: Collector RSS absent or above container limit`. Применён `lab6_backend_down_fix.zip` (изменение `tests/observability_live.py`, не бизнес-кода). Исправленный тест принимает нормализованные имена RSS, различает отсутствующие метрики и реальные превышения, измеряет память/очередь и проверяет состояние контейнера. Подтверждённый повторный вывод:
```text
Collector baseline: RSS=197.35 MiB, queue=0/64, limit=256 MiB
Collector outage: sampled RSS max=200.43 MiB, queue max=18/64, successful writes=16
PASS backend-down: writes and image processing persisted, telemetry failed visibly; queue/RAM bounded, fresh trace recovered
Saved evidence\lab6-backend-down.json
```
RSS — наблюдаемый максимум выборки, не абсолютный пик. 64 — вместимость очереди конкретного exporter, а не размер RabbitMQ. Успешная работа при одном коротком отказе не гарантирует отсутствие потерь телеметрии при произвольной длительности отказа. Числовую дельту failed exports следует брать из оригинального JSON на стенде.
## Регрессия Lab5 на стенде Lab6
| Команда | Результат |
| --- | --- |
| `python tests/own_unit.py` | PASS: models, cursor scoping, limits, Unicode U+0000 |
| `python tests/attachments_negative.py` | PASS: MIME, размер, имя файла, ACL, варианты и cross-chat |
| `python tests/idempotency_parallel.py` | PASS: две API-реплики, одно сообщение, replay 201, conflicting body 409 |
| `python tests/pipeline_resilience.py --scenario paused` | PASS: durable queued → ready, оригинал и 3 варианта, без дубликатов |
| `python tests/pipeline_resilience.py --scenario outbox-crash` | PASS: dispatcher exit 78 после commit outbox, до publish; восстановление queued → ready |
| `python tests/pipeline_resilience.py --scenario worker-crash` | PASS: worker exit 77 после S3 effects, до result/ack; восстановление queued → ready, без дубликатов |
Для трёх fault-тестов использовано `$env:LAB5_ALLOW_FAILURE_TEST = "yes"`.
## Нагрузка
`python tests/attachments_concurrent.py --concurrency 10 --output evidence/attachments-concurrent.json`
| Показатель | Фактическое значение |
| --- | ---: |
| Параллельных загрузок | 10 |
| HTTP 202 принятых | 10 |
| Вложений ready | 10 |
| Ошибок запроса | 0 |
| Вариантов на вложение | 3 |
| Размер исходного файла, B | 111817 |
| Приём всех запросов, s | 2.515 |
| Acceptance RPS | 3.976 |
| Upload latency p50, ms | 2296.5 |
| Upload latency max, ms | 2484.0 |
| Все вложения ready через, s | 36.406 |
Acceptance RPS не является throughput worker. 36.406 s — общее время до готовности всех десяти вложений, а не latency каждого из них. Данный эксперимент не измеряет предельную пропускную способность; p95/p99 на dashboard относятся к histogram разных наблюдаемых запросов, не вычисляются из p50/max этой таблицы.
## Grafana: исходные скриншоты
1. [API throughput, 5xx, latency, jobs](grafana/01-api-and-jobs.png).
2. [Worker duration, time to ready, queue depth, oldest age](grafana/02-worker-and-queue.png).
3. [DB pool, worker saturation, CPU, RAM](grafana/03-pool-and-resources.png).
4. [Collector queue, failed exports, alerts, SLI/SLO](grafana/04-collector-alerts-slo.png).
По всем 16 панелям видны реальные ряды. При контролируемой деградации видны рост 5xx и latency, накопление очереди и возраст задания, ошибки экспорта Tempo, нарушение цели SLO 99% с последующим восстановлением. Снимки дают визуальное подтверждение, а точные состояния firing/resolved и корреляцию logs/spans подтверждают автоматизированные сценарии выше. При отсутствии job events histogram может не иметь данных на отдельных интервалах — это нормально. CPU измеряется в ядрах, RAM — cgroup usage или RSS fallback согласно названию панели.
## Границы доказательств и дальнейшие проверки
- Шесть новых сценариев Lab6 и перечисленные регрессионные проверки подтверждены фактическими результатами. Оценку выставляет преподаватель.
- Отдельный closed-loop `tests/load.py`, общие persistence/failover `tests/resilience.py` и `pipeline_extended.py` (broker-persistence, failed-replay, staging-cleanup) для Lab6 не подтверждены предоставленным выводом. Их успехи в Lab5 не перенесены в итоги Lab6.
- Не измеряли максимальную длительность отказа telemetry, действительный peak RSS, полный объём потерянных spans и SLA на стороне клиента; счётчики failed exports доступны в данных стенда.
- JSON-файлы из таблиц уже созданы на машине автора; эта сводка их не заменяет и не воспроизводит отсутствующие поля.
- Для тестирования использовался `.env`, скопированный из `.env.example` без замены значений. Перед новым развёртыванием заменить примерные секреты; действующий `.env` не коммитить. Данные локальны, диагностика доступна на loopback.
+4
View File
@@ -0,0 +1,4 @@
# Persistent SeaweedFS filer metadata for the single-node lab stack.
[leveldb2]
enabled = true
dir = "/data/filerldb2"
+32
View File
@@ -0,0 +1,32 @@
global
log stdout format raw local0
maxconn 1000
defaults
log global
mode http
option httplog
option dontlognull
timeout connect 2s
timeout client 60s
timeout server 60s
timeout http-request 5s
resolvers docker
nameserver localdns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
frontend messenger
bind *:8080
default_backend api_nodes
backend api_nodes
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server inter 1s fall 2 rise 1 slowstart 2s
server api1 api1:8080 check resolvers docker init-addr none
server api2 api2:8080 check resolvers docker init-addr none
+71
View File
@@ -0,0 +1,71 @@
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 180
spike_limit_mib: 40
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 0.0.0.0:8889
metric_expiration: 15s
add_metric_suffixes: true
resource_to_telemetry_conversion:
enabled: true
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true
timeout: 3s
sending_queue:
enabled: true
num_consumers: 1
queue_size: 64
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 3s
max_elapsed_time: 10s
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 3s
sending_queue:
enabled: true
num_consumers: 1
queue_size: 64
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 3s
max_elapsed_time: 10s
service:
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp/tempo]
@@ -0,0 +1,805 @@
{
"uid": "lab6-messenger",
"title": "Lab6 | Messenger observability",
"schemaVersion": 39,
"version": 1,
"editable": false,
"timezone": "browser",
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"tags": [
"lab6",
"otel"
],
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "Business API throughput",
"description": "Excludes health, counts completed responses.",
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s]))",
"legendFormat": "all",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "B",
"expr": "sum(rate(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[30s])) or vector(0)",
"legendFormat": "5xx",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 2,
"type": "timeseries",
"title": "Business API error ratio (5xx)",
"description": "5xx / all business HTTP responses; threshold 5%.",
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "(sum(rate(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s])),0.001)",
"legendFormat": "5xx / all",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 3,
"type": "timeseries",
"title": "API latency p50/p95/p99",
"description": "Aggregated histogram buckets across API replicas; seconds, time to response headers.",
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])) by (le))",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])) by (le))",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])) by (le))",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 4,
"type": "timeseries",
"title": "Jobs per second by outcome",
"description": "Attempt outcomes: ready, retry, failed, stale, requeue; not exactly-once audit counts.",
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "sum by(outcome)(rate(lab_jobs_total[30s]))",
"legendFormat": "{{outcome}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 5,
"type": "timeseries",
"title": "Worker processing duration",
"description": "",
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum(rate(lab_job_duration_seconds_bucket[30s])) by (le))",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum(rate(lab_job_duration_seconds_bucket[30s])) by (le))",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum(rate(lab_job_duration_seconds_bucket[30s])) by (le))",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 6,
"type": "timeseries",
"title": "Time to ready",
"description": "Acceptance to durable ready (includes queue, retry and processing).",
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum(rate(lab_ready_duration_seconds_bucket[30s])) by (le))",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum(rate(lab_ready_duration_seconds_bucket[30s])) by (le))",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum(rate(lab_ready_duration_seconds_bucket[30s])) by (le))",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 7,
"type": "timeseries",
"title": "Queue depth: durable backlog / RabbitMQ ready",
"description": "DB queued includes delayed/unpublished tasks; RabbitMQ ready counts published messages waiting for delivery; neither includes busy jobs.",
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "max(lab_queue_depth)",
"legendFormat": "DB queued",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "B",
"expr": "max(lab_broker_ready)",
"legendFormat": "broker ready",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 8,
"type": "timeseries",
"title": "Oldest queued job age",
"description": "",
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "max(lab_queue_oldest_age_seconds)",
"legendFormat": "age",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 9,
"type": "timeseries",
"title": "Database pool",
"description": "",
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "lab_db_pool_used",
"legendFormat": "used {{service_instance_id}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "B",
"expr": "lab_db_pool_waiting",
"legendFormat": "waiting {{service_instance_id}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "C",
"expr": "lab_db_pool_capacity",
"legendFormat": "capacity {{service_instance_id}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 10,
"type": "timeseries",
"title": "Worker saturation",
"description": "",
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "sum(lab_worker_busy) / clamp_min(sum(lab_worker_capacity),1)",
"legendFormat": "busy / capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 11,
"type": "timeseries",
"title": "Container memory (or tree RSS fallback)",
"description": "",
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "lab_container_memory_bytes",
"legendFormat": "{{service_name}} / {{service_instance_id}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 12,
"type": "timeseries",
"title": "Container CPU",
"description": "CPU cores: 1.0 = one logical core.",
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "lab_container_cpu_cores",
"legendFormat": "{{service_name}} / {{service_instance_id}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 13,
"type": "timeseries",
"title": "Collector exporter queue",
"description": "",
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "otelcol_exporter_queue_size",
"legendFormat": "{{exporter}} queue",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "B",
"expr": "otelcol_exporter_queue_capacity",
"legendFormat": "{{exporter}} capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 14,
"type": "timeseries",
"title": "Collector failed exports",
"description": "",
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "rate(otelcol_exporter_send_failed_spans[30s]) or rate(otelcol_exporter_send_failed_spans_total[30s])",
"legendFormat": "spans {{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "B",
"expr": "rate(otelcol_exporter_send_failed_log_records[30s]) or rate(otelcol_exporter_send_failed_log_records_total[30s])",
"legendFormat": "logs {{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 15,
"type": "timeseries",
"title": "Alert state",
"description": "",
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "ALERTS{alertname=~\"Lab.*\"}",
"legendFormat": "{{alertname}} / {{alertstate}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
},
{
"id": 16,
"type": "timeseries",
"title": "SLI: successful availability (5 min)",
"description": "1 \u2212 5xx/all business responses over 5 min; objective >= 99%. No data if no traffic.",
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
}
},
"targets": [
{
"refId": "A",
"expr": "1 - ((sum(increase(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / sum(increase(lab_http_requests_total{route=~\"/api/.*\"}[5m])))",
"legendFormat": "actual SLI",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
},
{
"refId": "B",
"expr": "vector(0.99)",
"legendFormat": "SLO target 99%",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code"
}
]
}
]
}
@@ -0,0 +1,11 @@
apiVersion: 1
providers:
- name: Lab6
orgId: 1
folder: MTUCI
type: file
disableDeletion: true
allowUiUpdates: false
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,36 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
- name: Loki
type: loki
uid: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: TraceID
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: '$${__value.raw}'
- name: Tempo
type: tempo
uid: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: '-2s'
spanEndTimeShift: '2s'
filterByTraceID: true
filterBySpanID: false
tags:
- key: service.name
value: service_name
+32
View File
@@ -0,0 +1,32 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 72h
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+19
View File
@@ -0,0 +1,19 @@
groups:
- name: messenger-lab
rules:
- alert: LabAPIHighErrorRate
expr: (sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])), 0.001) > 0.05
for: 9s
labels:
severity: warning
annotations:
summary: Business HTTP 5xx rate exceeds 5% over 30 seconds
runbook: REPORT.md#инциденты-и-runbook
- alert: LabQueueLag
expr: max(lab_queue_oldest_age_seconds) > 15 and max(lab_queue_depth) > 0
for: 12s
labels:
severity: warning
annotations:
summary: Oldest queued job exceeds 15 seconds (sustained 12 seconds)
runbook: REPORT.md#инциденты-и-runbook
+12
View File
@@ -0,0 +1,12 @@
global:
scrape_interval: 3s
evaluation_interval: 3s
rule_files:
- /etc/prometheus/alerts.yml
scrape_configs:
- job_name: lab-otlp
static_configs:
- targets: ['otel-collector:8889']
- job_name: collector-self
static_configs:
- targets: ['otel-collector:8888']
+22
View File
@@ -0,0 +1,22 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
compactor:
compaction:
block_retention: 72h
usage_report:
reporting_enabled: false
+18
View File
@@ -0,0 +1,18 @@
fastapi==0.115.12
uvicorn==0.34.2
pydantic==2.11.4
psycopg[binary]==3.2.6
psycopg-pool==3.2.6
redis==5.2.1
argon2-cffi==23.1.0
boto3==1.40.0
Pillow==11.3.0
pika==1.3.2
opentelemetry-api==1.42.1
opentelemetry-sdk==1.42.1
opentelemetry-exporter-otlp-proto-http==1.42.1
psutil==7.2.2
PyYAML==6.0.2
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Additional static checks (not the teacher's make check and not a live deploy)."""
import ast,json,re
from pathlib import Path
import yaml
ROOT=Path(__file__).resolve().parents[1]
def main():
count=0
for path in (ROOT/'app').glob('*.py'):
tree=ast.parse(path.read_text(encoding='utf-8'))
for n in ast.walk(tree):
if isinstance(n,ast.Call) and isinstance(n.func,ast.Attribute) and n.func.attr=='execute' and len(n.args)>1 and isinstance(n.args[0],ast.Constant) and isinstance(n.args[0].value,str) and isinstance(n.args[1],(ast.Tuple,ast.List)):
assert n.args[0].value.count('%s')==len(n.args[1].elts),f'SQL binding mismatch: {path.name}:{n.lineno}'
count+=1
compose=yaml.safe_load((ROOT/'compose.yaml').read_text())
for name,svc in compose['services'].items():
if 'image' in svc:assert ':' in svc['image'] and not svc['image'].endswith(':latest'),name
for port in svc.get('ports',[]):assert port.startswith('127.0.0.1:'),name
for mount in svc.get('volumes',[]):
src=mount.split(':')[0]
assert (ROOT/src).exists() if src.startswith('./') else src in compose['volumes'],f'missing mount {src}'
for network in svc.get('networks',[]):assert network in compose['networks']
for dep in svc.get('depends_on',{}):assert dep in compose['services']
for p in (ROOT/'infra/otel').rglob('*'):
if p.suffix in ('.yml','.yaml'):yaml.safe_load(p.read_text())
services=compose['services']; assert all('telemetry' in services[s]['networks'] for s in ('api1','api2','dispatcher','worker','otel-collector','prometheus','loki','tempo','grafana'))
for v in ('prometheus_data','loki_data','tempo_data','grafana_data'):assert v in compose['volumes']
collector=yaml.safe_load((ROOT/'infra/otel/collector.yaml').read_text())
for signal in ('metrics','logs','traces'):
pipe=collector['service']['pipelines'][signal]
assert pipe['receivers']==['otlp'] and pipe['processors']==['memory_limiter','batch']
assert all(e in collector['exporters'] for e in pipe['exporters'])
for exp in ('otlp/tempo','otlphttp/loki'):
conf=collector['exporters'][exp];assert conf['sending_queue']['queue_size']==64 and conf['retry_on_failure']['max_elapsed_time']=='10s'
dashboard=json.loads((ROOT/'infra/otel/grafana/dashboards/messenger.json').read_text())
expressions='\n'.join(t['expr'] for p in dashboard['panels'] for t in p['targets'])
for metric in ('lab_http_requests_total','lab_http_duration_seconds_bucket','lab_jobs_total','lab_job_duration_seconds_bucket','lab_ready_duration_seconds_bucket','lab_queue_depth','lab_queue_oldest_age_seconds','lab_broker_ready','lab_db_pool_used','lab_db_pool_waiting','lab_worker_busy','lab_worker_capacity','lab_container_cpu_cores','lab_container_memory_bytes'):
assert metric in expressions,metric
assert all(f'histogram_quantile({q}' in expressions for q in (.5,.95,.99))
rules=yaml.safe_load((ROOT/'infra/otel/prometheus/alerts.yml').read_text())
assert {x['alert'] for g in rules['groups'] for x in g['rules']}=={'LabAPIHighErrorRate','LabQueueLag'}
assert all(x.get('for') and x['annotations'].get('runbook') for g in rules['groups'] for x in g['rules'])
print(f'PASS: {count} literal SQL bindings; pinned Compose images, mounts and networks; 3 OTLP pipelines, dashboards and alert rules (static only)')
if __name__=='__main__':main()
+11
View File
@@ -0,0 +1,11 @@
@echo off
setlocal
cd /d "%~dp0.."
docker compose --env-file .env stop --timeout 5 lb api1 api2 worker dispatcher broker s3 redis postgres grafana otel-collector prometheus loki tempo
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --wait postgres redis broker s3 s3-bootstrap prometheus loki tempo otel-collector grafana
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --no-deps api1 api2 lb dispatcher worker
if errorlevel 1 exit /b 1
python scripts\wait_ready.py
exit /b %ERRORLEVEL%
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Preserve named volumes. Restart all state stores and both API processes.
docker compose --env-file .env stop --timeout 5 lb api1 api2 worker dispatcher broker s3 redis postgres grafana otel-collector prometheus loki tempo
docker compose --env-file .env up -d --wait postgres redis broker s3 s3-bootstrap prometheus loki tempo otel-collector grafana
docker compose --env-file .env up -d --no-deps api1 api2 lb dispatcher worker
python - <<'PY'
import time, urllib.request
for i in range(120):
try:
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2) as response:
if response.status == 200:
print('[restart] all services ready')
break
except Exception:
pass
time.sleep(.5)
else:
raise SystemExit('[restart] health timeout')
PY
+10
View File
@@ -0,0 +1,10 @@
@echo off
setlocal
cd /d "%~dp0.."
if "%TARGET_INSTANCE_ID%"=="api1" goto stop
if "%TARGET_INSTANCE_ID%"=="api2" goto stop
echo Unexpected TARGET_INSTANCE_ID 1>&2
exit /b 2
:stop
docker compose --env-file .env stop --timeout 1 %TARGET_INSTANCE_ID%
exit /b %ERRORLEVEL%
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
# The public failover test sets TARGET_INSTANCE_ID from the actual response header.
set -eu
cd "$(dirname "$0")/.."
case "${TARGET_INSTANCE_ID:?missing TARGET_INSTANCE_ID}" in
api1|api2) docker compose --env-file .env stop --timeout 1 "$TARGET_INSTANCE_ID" ;;
*) echo 'Unexpected instance alias' >&2; exit 2 ;;
esac
+14
View File
@@ -0,0 +1,14 @@
"""Operator-facing readiness wait after container replacement."""
import time
import urllib.request
for _ in range(120):
try:
with urllib.request.urlopen("http://localhost:8080/health/ready", timeout=2) as response:
if response.status == 200:
print("[restart] all services ready")
break
except Exception:
pass
time.sleep(.5)
else:
raise SystemExit("[restart] health timeout")
+90
View File
@@ -0,0 +1,90 @@
#!/usr/bin/env python3
"""Concurrent Lab5 uploads (HTTP 202) and eventual processing completion.
Admission metrics measure upload/enqueue requests, not worker throughput.
Completion is checked separately so an accepted-but-stuck job cannot pass.
"""
import argparse
import concurrent.futures
import json
import statistics
import time
from pathlib import Path
from client import Client, check, run
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--output', default='evidence/attachments-concurrent.json')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--wait-seconds', type=int, default=180)
args = parser.parse_args()
check(args.concurrency > 0, 'concurrency must be positive')
check(args.wait_seconds > 0, 'wait-seconds must be positive')
client = Client(args.base_url)
alice, bob, eve, chat = client.bootstrap()
source = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=parallel.png'
def upload(_):
cli = Client(args.base_url)
response = cli.call('POST', path, 202, actor=alice, raw=source,
headers={'Content-Type': 'image/png'}, schema='Attachment')
att = response['json']
check(att['status'] == 'queued', '202 must return a queued attachment')
check(att['size_bytes'] == len(source), 'size mismatch')
check(att['chat_id'] == chat['id'] and att['owner_id'] == alice['id'],
'incorrect attachment ownership')
return att['id'], response['elapsed']
started = time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
results = list(pool.map(upload, range(args.concurrency)))
accepted_seconds = time.monotonic() - started
ids = [att_id for att_id, _ in results]
timings = [duration for _, duration in results]
check(len(set(ids)) == args.concurrency, 'duplicate attachment IDs')
# A 202 response alone only proves acceptance, not successful processing.
pending = set(ids)
deadline = time.monotonic() + args.wait_seconds
while pending and time.monotonic() < deadline:
for att_id in list(pending):
att = client.call('GET', '/api/v1/attachments/' + att_id,
actor=alice, schema='Attachment')['json']
check(att['status'] != 'failed', 'attachment reached failed: ' + att_id)
if att['status'] == 'ready':
variants = [v['name'] for v in att['variants']]
check(len(variants) == 3 and set(variants) == {'thumbnail', 'crop', 'optimized'},
'missing/duplicate image variants: ' + att_id)
pending.remove(att_id)
if pending:
time.sleep(0.5)
check(not pending, 'attachments not ready before deadline: ' + str(len(pending)))
complete_seconds = time.monotonic() - started
data = {
'concurrency': args.concurrency,
'accepted_uploads': len(results),
'ready_attachments': args.concurrency,
'failed_requests': 0,
'acceptance_elapsed_seconds': round(accepted_seconds, 3),
'acceptance_rps': round(len(results) / accepted_seconds, 3),
'upload_latency_p50_ms': round(statistics.median(timings) * 1000, 1),
'upload_latency_max_ms': round(max(timings) * 1000, 1),
'all_ready_elapsed_seconds': round(complete_seconds, 3),
'bytes_per_file': len(source),
'note': 'HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats.',
}
output = Path(args.output)
output.parent.mkdir(exist_ok=True, parents=True)
output.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
print('PASS: %d concurrent HTTP 202 uploads -> ready, 3 variants each' % args.concurrency)
print(json.dumps(data, ensure_ascii=False, indent=2))
if __name__ == '__main__':
run(main)
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Lab5 negative API tests with isolated synthetic users (requires live stack)."""
from urllib.parse import quote
import uuid
from client import Client, arguments, run, check
from pathlib import Path
def main():
args = arguments('Lab5 negative file/ACL/security tests')
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
base = '/api/v1/chats/' + chat['id'] + '/attachments?filename='
data = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
c.call('POST', base+'big.png', 413, actor=alice, raw=data+b'\x00'*(10*1024*1024+1-len(data)), headers={'Content-Type':'image/png'})
c.call('POST', base+'bad.png', 415, actor=alice, raw=b'not-an-image', headers={'Content-Type':'image/png'})
c.call('POST', base+'fake.jpg', 415, actor=alice, raw=data, headers={'Content-Type':'image/jpeg'})
c.call('POST', base+quote('../escape.png',safe=''), 400, actor=alice, raw=data, headers={'Content-Type':'image/png'})
c.call('POST', base+'x.png', 404, actor=eve, raw=data, headers={'Content-Type':'image/png'})
c.call('POST', base+'x.png', 401, raw=data, headers={'Content-Type':'image/png'})
uploaded = c.attachment(alice,chat)[0]
ready = c.wait_ready(alice, uploaded)
unknown = str(uuid.uuid4())
c.call('GET','/api/v1/attachments/'+unknown,404,actor=alice)
c.call('GET','/api/v1/attachments/'+uploaded['id'],404,actor=eve)
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',404,actor=eve)
# A ready Lab5 attachment has a downloadable thumbnail for chat members.
# A non-member still gets 404, and an unknown variant must not resolve.
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',404,actor=eve)
c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=nonexistent',404,actor=alice)
check(any(v['name'] == 'thumbnail' for v in ready['variants']), 'missing thumbnail variant')
thumb = c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',200,actor=alice)
check(thumb['headers'].get_content_type() == 'image/jpeg', 'wrong thumbnail MIME')
other_chat = c.call('POST','/api/v1/chats',201,actor=alice,
body={'title':'Другой чат','member_ids':[alice['id'],bob['id']]},schema='Chat')['json']
path = '/api/v1/chats/'+other_chat['id']+'/messages'
c.call('POST',path,404,actor=alice,
body={'text':'cross-chat','attachment_ids':[uploaded['id']]},
headers={'Idempotency-Key':uuid.uuid4().hex})
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',404,actor=alice,
body={'text':'unknown','attachment_ids':[unknown]},
headers={'Idempotency-Key':uuid.uuid4().hex})
ok = c.message(bob,chat,'',attachments=[uploaded['id']])['json']
check(ok['attachment_ids']==[uploaded['id']], 'valid attachment not linked')
print('PASS: lab5 negative image, MIME, size, filename, ACL, variants and cross-chat checks')
if __name__=='__main__':
run(main)
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Lab3 own test: 30 requests using identical actor/chat/key/body across both replicas."""
import concurrent.futures
import sys
import uuid
from client import Client, Failure, arguments, check, run
def main():
args = arguments("Check atomic idempotency across API replicas")
c = Client(args.base_url, args.ca_file)
alice, bob, _, chat = c.bootstrap()
key = uuid.uuid4().hex
body = {"text": "Concurrent replay " + key}
path = f"/api/v1/chats/{chat['id']}/messages"
def worker(_):
# Independent request client; one shared bearer token and idempotency key.
client = Client(args.base_url, args.ca_file)
return client.call("POST", path, expected=201, actor=alice, body=body,
headers={"Idempotency-Key": key}, schema="Message")
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor:
results = list(executor.map(worker, range(30)))
ids = {result["json"]["id"] for result in results}
instances = {result["headers"]["X-Instance-Id"] for result in results}
check(len(ids) == 1, "parallel replay created different messages")
check(len(instances) >= 2, "LB did not serve requests via both API replicas")
check(all(r["json"] == results[0]["json"] for r in results), "replay returned altered result")
c.call("POST", path, 409, actor=alice, body={"text": "different body"}, headers={"Idempotency-Key": key})
c.call("POST", path, 201, actor=bob, body=body, headers={"Idempotency-Key": key}, schema="Message")
history = c.call("GET", path + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"]
check(sum(x["id"] in ids for x in history) == 1, "message duplicated in history")
print("PASS: parallel idempotency across", len(instances), "replicas; one message, replay 201, conflicting body 409")
if __name__ == "__main__":
run(main)
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env python3
"""Prevent a mismatch between SQL placeholders and parameter tuple in worker result persistence."""
import ast
from pathlib import Path
import re
source = (Path(__file__).resolve().parents[1] / "app" / "jobs.py").read_text(encoding="utf-8")
tree = ast.parse(source)
fn = next(x for x in tree.body if isinstance(x, ast.FunctionDef) and x.name == "publish_result")
matches = []
for node in ast.walk(fn):
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
continue
if node.func.attr != "execute" or len(node.args) < 2:
continue
if not isinstance(node.args[0], ast.Constant) or not isinstance(node.args[0].value, str):
continue
sql = node.args[0].value
if "INSERT INTO attachment_variants" not in sql:
continue
assert isinstance(node.args[1], ast.Tuple), "Expected literal parameters tuple"
placeholders = sql.count("%s")
params = len(node.args[1].elts)
columns_match = re.search(r"INSERT INTO attachment_variants\s*\(([^)]*)\)", sql, re.I | re.S)
assert columns_match, "Columns missing in INSERT"
columns = len(columns_match.group(1).split(","))
assert placeholders == columns == params, f"SQL mismatch: {columns} columns, {placeholders} placeholders, {params} parameters"
matches.append((columns, placeholders, params))
assert len(matches) == 1, "Expected exactly one attachment_variants INSERT"
print("PASS: attachment_variants INSERT has 9 columns, 9 placeholders, 9 arguments")
+341
View File
@@ -0,0 +1,341 @@
#!/usr/bin/env python3
"""Lab6 live evidence. Only synthetic test stack; no volume removal.
Run scenarios separately. Queries stored in evidence, no HTTP auth or payloads.
"""
import argparse
import base64
import math
import concurrent.futures
import json
import os
import subprocess
import time
import urllib.parse
import urllib.request
import urllib.error
from pathlib import Path
from client import Client, Failure, check, run
from pipeline_extended import compose, pg, scalar, wait_for
from pipeline_resilience import verify_ready
ROOT=Path(__file__).resolve().parents[1]
PROM='http://127.0.0.1:9090'
LOKI='http://127.0.0.1:3100'
TEMPO='http://127.0.0.1:3200'
GRAFANA='http://127.0.0.1:3000'
ERR='(sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])),0.001)'
LAG='max(lab_queue_oldest_age_seconds)'
DEPTH='max(lab_queue_depth)'
FAILED='sum(otelcol_exporter_send_failed_spans) or sum(otelcol_exporter_send_failed_spans_total) or vector(0)'
# Explicit Prometheus readers can append unit suffixes to Collector self-metrics.
# Missing gauges are not zero: absent telemetry must fail the outage test.
QUEUE='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_size(_batches|_requests|_items)?"})'
QUEUE_CAPACITY='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_capacity(_batches|_requests|_items)?"})'
RSS='max({job="collector-self",__name__=~"otelcol_process_memory_rss(_bytes)?"})'
def get(url, params=None, *, as_json=True):
try:
with urllib.request.urlopen(url+('?' + urllib.parse.urlencode(params) if params else ''),timeout=8) as r:
raw=r.read(32*1024*1024)
return json.loads(raw) if as_json else raw.decode('utf-8',errors='replace')
except (urllib.error.URLError,TimeoutError,ValueError) as exc:
raise Failure('diagnostic endpoint unavailable: '+url.split('?')[0]+' ('+type(exc).__name__+')') from None
def prom(expr):
data=get(PROM+'/api/v1/query',{'query':expr})
check(data.get('status')=='success','PromQL query failed')
return data['data']['result']
def val(expr):
rows=prom(expr)
return float(rows[0]['value'][1]) if rows else None
def alert_state(name):
rules=get(PROM+'/api/v1/rules')['data']['groups']
for group in rules:
for rule in group['rules']:
if rule['name']==name:
check(rule.get('health')=='ok','alert rule is not healthy: '+name)
return rule['state']
raise Failure('alert rule missing: '+name)
def wait_alert(name,state,seconds=100):
return wait_for(name+' '+state,lambda:alert_state(name),lambda x:x==state,seconds=seconds,interval=2)
def ready():
for url in (PROM+'/-/ready',LOKI+'/ready',TEMPO+'/ready'):
wait_for(url,lambda:get(url,as_json=False),lambda x:bool(x),seconds=100,interval=2)
wait_for('Grafana',lambda:get(GRAFANA+'/api/health'),lambda x:x.get('database')=='ok',seconds=100)
wait_for('Collector scrape',lambda:val('up{job="collector-self"}'),lambda v:v==1,seconds=40)
wait_for('OTLP metrics scrape',lambda:val('up{job="lab-otlp"}'),lambda v:v==1,seconds=40)
def trace_spans(tid):
raw=get(TEMPO+'/api/traces/'+tid)
spans=[]
def walk(obj):
if isinstance(obj,dict):
if {'spanId','traceId','name'}<=set(obj): spans.append(obj)
else:
for v in obj.values():walk(v)
elif isinstance(obj,list):
for v in obj:walk(v)
walk(raw)
return spans
def logs(tid,service,request_id=None):
expr='{service_name="%s"} | trace_id="%s"' % (service,tid)
if request_id: expr+=' | request_id="%s"'%request_id
data=get(LOKI+'/loki/api/v1/query_range',{'query':expr,'start':str(int((time.time()-900)*1e9)), 'end':str(time.time_ns()),'limit':100})
return data['data']['result']
def upload(c,alice,chat):
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
r=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
tid=r['headers'].get('X-Trace-Id','')
check(len(tid)==32 and int(tid,16)>0,'missing active API trace ID: check SDK enablement')
return r['json'],raw,tid,r['headers']['X-Request-Id']
def span_attrs(s):
attrs={}
for a in s.get('attributes',[]):
value=a.get('value',{})
attrs[a['key']]=next(iter(value.values()),None)
return attrs
def chain_ok(spans):
names={s['name'] for s in spans}
required={'POST /api/v1/chats/{chat_id}/attachments','rabbitmq.publish','rabbitmq.consume','s3.upload_fileobj','s3.get_object','s3.put_object','db.publish_result'}
if not required<=names:return False
producers={s['spanId'] for s in spans if s['name']=='rabbitmq.publish'}
consumers=[s for s in spans if s['name']=='rabbitmq.consume']
outcomes={span_attrs(s).get('job.outcome') for s in consumers}
return len(producers)>=2 and {'retry','ready'}<=outcomes and all(s.get('parentSpanId') in producers for s in consumers)
def trace_retry(c):
alice,bob,_,chat=c.bootstrap()
compose('stop','worker')
try:
att,raw,tid,rid=upload(c,alice,chat)
# Override only worker, never a public endpoint; reset in finally.
env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'once'}
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env=env,check=True,timeout=180)
verify_ready(c,bob,att,raw,seconds=180)
spans=wait_for('stored retry trace',lambda:trace_spans(tid),chain_ok,seconds=100,interval=3)
wait_for('API log joined by request/trace ID',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
def worker_events(streams):
return {v[1] for row in streams for v in row.get('values',[])}
wlogs=wait_for('worker retry and ready logs',lambda:logs(tid,'messenger-worker'),lambda rows:{'job.retry','job.ready'}<=worker_events(rows),seconds=90)
# Keep only allowlisted names/IDs, not arbitrary span values or log payloads.
summary=[{'name':s['name'],'id':s['spanId'],'parent':s.get('parentSpanId'),'attempt':span_attrs(s).get('job.attempt'),'outcome':span_attrs(s).get('job.outcome')} for s in spans]
result={'trace_id':tid,'request_id':rid,'spans':summary,'worker_log_streams':len(wlogs),'worker_events':sorted(worker_events(wlogs)&{'job.claimed','job.retry','job.ready','image.transformed'}),'retry_chain':True}
print('PASS trace-retry: API/producer/consumer/S3/DB with retry, correlated logs',tid)
return result
finally:
subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'off'},check=True,timeout=180)
def queue_lag(c):
alice,bob,_,chat=c.bootstrap()
wait_alert('LabQueueLag','inactive')
compose('stop','worker')
try:
att,raw,tid,rid=upload(c,alice,chat)
t=time.monotonic()
wait_alert('LabQueueLag','firing')
lag=val(LAG);depth=val(DEPTH)
check(lag is not None and lag>15 and depth>=1,'firing not supported by actual backlog metrics')
detected=time.monotonic()-t
print('Queue alert firing: oldest age',lag,'s, queued',depth,flush=True)
compose('up','-d','worker')
verify_ready(c,bob,att,raw,seconds=180)
wait_alert('LabQueueLag','inactive',seconds=120)
wait_for('queue drained',lambda:val(DEPTH),lambda v:v==0,seconds=60,interval=2)
print('PASS queue-lag: firing -> ready -> resolved')
return {'trace_id':tid,'firing_lag_seconds':lag,'firing_depth':depth,'detected_seconds':round(detected,3),'resolved_depth':val(DEPTH)}
finally:compose('up','-d','worker')
def db_error(c):
alice,_,_,_=c.bootstrap()
wait_alert('LabAPIHighErrorRate','inactive')
user=compose('exec','-T','postgres','printenv','POSTGRES_USER')
db=compose('exec','-T','postgres','printenv','POSTGRES_DB')
# Bounded, isolated table lock. Health SELECT 1 works, authenticated business
# read times out at statement_timeout. Never stop DB (LB would hide API 5xx).
cmd=['docker','compose','--env-file','.env','exec','-T','postgres','psql','-X','-q','-v','ON_ERROR_STOP=1','-U',user,'-d',db,'-c','BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(60); COMMIT;']
lock=subprocess.Popen(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
try:
wait_for('users lock held',lambda:scalar("SELECT count(*) FROM pg_locks WHERE relation='users'::regclass AND mode='AccessExclusiveLock' AND granted"),lambda v:int(v)>0,seconds=15)
start=time.monotonic(); traces=[]; codes=[]
while time.monotonic()-start < 43:
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
def get_user(_):
r=Client(timeout=12).call('GET','/api/v1/users/me',[200,503],actor=alice,schema='User')
return r['status'],r['headers'].get('X-Trace-Id'),r['headers'].get('X-Request-Id')
for code,tid,rid in pool.map(get_user,range(3)):
codes.append(code)
if code==503 and tid:traces.append((tid,rid))
if traces and alert_state('LabAPIHighErrorRate')=='firing':break
check(bool(traces),'no real HTTP 503 from DB lock')
wait_alert('LabAPIHighErrorRate','firing',seconds=20)
ratio=val(ERR)
check(ratio is not None and ratio>0.05,'error ratio below threshold')
tid,rid=traces[0]
spans=wait_for('DB timeout trace',lambda:trace_spans(tid),lambda v:any(s['name']=='db.transaction' and span_attrs(s).get('error.type')=='QueryCanceled' for s in v),seconds=60,interval=2)
wait_for('error log',lambda:logs(tid,'messenger-api',rid),bool,seconds=60)
print('Error alert firing, database QueryCanceled visible; ratio',ratio,flush=True)
finally:
# SQL transaction has server-side 60s bound; wait for normal COMMIT.
try:lock.wait(timeout=75)
except subprocess.TimeoutExpired:
lock.terminate();lock.wait(timeout=10)
# Keep successful business traffic in the evaluation window to prove recovery.
t=time.monotonic()
while time.monotonic()-t<65:
c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
if alert_state('LabAPIHighErrorRate')=='inactive' and val(ERR) is not None and val(ERR)<=.05:break
time.sleep(1)
wait_alert('LabAPIHighErrorRate','inactive',seconds=40)
print('PASS db-error: lock -> QueryCanceled/503 -> firing -> resolved')
return {'trace_id':tid,'request_id':rid,'http_codes':codes,'firing_error_ratio':ratio,'resolved_error_ratio':val(ERR),'error_spans':[{'name':s['name'],'error_type':span_attrs(s).get('error.type')} for s in spans if span_attrs(s).get('error.type')]}
def collector_sample():
check(val('up{job="collector-self"}')==1,'Collector self-metrics scrape down')
rss,queue,capacity=val(RSS),val(QUEUE),val(QUEUE_CAPACITY)
check(rss is not None and math.isfinite(rss) and rss>0,
'Collector RSS metric absent or invalid; query otelcol_process_memory_rss(_bytes)? in Prometheus')
check(queue is not None and math.isfinite(queue) and queue>=0,
'Tempo exporter queue size metric absent or invalid; inspect collector-self metrics')
check(capacity is not None and math.isfinite(capacity) and capacity==64,
'Tempo exporter queue capacity absent or differs from configured 64: '+str(capacity))
return rss,queue,capacity
def backend_down(c):
alice,bob,_,chat=c.bootstrap()
baseline=val(FAILED) or 0
cid=compose('ps','-q','otel-collector')
before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
memory_limit=int(subprocess.check_output(['docker','inspect','--format','{{.HostConfig.Memory}}',cid],text=True).strip())
check(memory_limit==256*1024*1024,'unexpected Collector container memory limit: '+str(memory_limit))
rss,queue,capacity=collector_sample()
print(f'Collector baseline: RSS={rss/1024**2:.2f} MiB, queue={queue:g}/{capacity:g}, limit={memory_limit/1024**2:g} MiB',flush=True)
compose('stop','tempo')
try:
start=time.monotonic();n=0;memory=[];queues=[];failures=[]
att,raw,tid,_=upload(c,alice,chat)
while time.monotonic()-start<40:
c.message(alice,chat,'telemetry-backend-offline')
n+=1
rss,queue,capacity=collector_sample()
memory.append(rss);queues.append(queue)
x=val(FAILED)
if x is not None:failures.append(x)
time.sleep(.3)
verify_ready(c,bob,att,raw,seconds=180)
check(n>0,'no successful writes during backend outage')
check(failures and max(failures)>baseline,'no recorded exporter failure; inspect Collector internal metrics')
rss,queue,capacity=collector_sample()
memory.append(rss);queues.append(queue)
print(f'Collector outage: sampled RSS max={max(memory)/1024**2:.2f} MiB, queue max={max(queues):g}/{capacity:g}, successful writes={n}',flush=True)
after=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip()
check(after==before and after.endswith('false'),'Collector restarted or OOM-killed: '+before+' -> '+after)
check(max(memory)<memory_limit,f'Collector RSS {max(memory):g} bytes exceeds/equal to container limit {memory_limit} bytes')
check(max(queues)<=capacity,'collector queue exceeds configured cap')
result={'successful_messages_during_tempo_outage':n,'trace_id':tid,'max_sampled_collector_rss_bytes':max(memory),'collector_memory_limit_bytes':memory_limit,'rss_sample_count':len(memory),'max_export_queue':max(queues),'export_queue_capacity':capacity,'failed_span_counter_delta':max(failures)-baseline,'restart_state':after}
finally:compose('up','-d','tempo')
ready()
# Confirm a fresh post-recovery trace can be retrieved.
r=c.call('GET','/api/v1/users/me',200,actor=alice,schema='User')
tid2=r['headers']['X-Trace-Id']
wait_for('post-recovery trace',lambda:trace_spans(tid2),bool,seconds=90)
result['recovered_trace_id']=tid2
print('PASS backend-down: writes and image processing persisted, telemetry failed visibly; queue/RAM bounded, fresh trace recovered')
return result
def metrics_snapshot(c):
alice,bob,_,chat=c.bootstrap()
att,raw,_,_=upload(c,alice,chat)
verify_ready(c,bob,att,raw,seconds=150)
until=time.monotonic()+12
while time.monotonic()<until:
c.message(alice,chat,'synthetic-metric-check')
time.sleep(5)
# Read the real provisioned Grafana dashboard/datasource list, without
# printing credentials or copying the response into the evidence.
secret=compose('exec','-T','grafana','printenv','GF_SECURITY_ADMIN_PASSWORD')
auth=base64.b64encode(('admin:'+secret).encode()).decode()
def grafana_json(path):
r=urllib.request.Request(GRAFANA+path,headers={'Authorization':'Basic '+auth})
try:
with urllib.request.urlopen(r,timeout=8) as f:return json.load(f)
except (urllib.error.URLError,TimeoutError,ValueError):
raise Failure('Grafana authenticated provisioning check failed; verify admin password/volume') from None
dashboard=grafana_json('/api/dashboards/uid/lab6-messenger')['dashboard']
check(len(dashboard['panels'])>=15,'provisioned dashboard incomplete')
check({'prometheus','loki','tempo'}<={d['uid'] for d in grafana_json('/api/datasources')},'provisioned datasources missing')
queries={'api_rps':'sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))','error_ratio':ERR,
'p50':'histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
'p95':'histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
'p99':'histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))',
'jobs':'lab_jobs_total','job_histogram':'lab_job_duration_seconds_count','ready_histogram':'lab_ready_duration_seconds_count',
'depth':DEPTH,'oldest':LAG,'broker_ready':'lab_broker_ready','broker_consumers':'lab_broker_consumers','pool_used':'lab_db_pool_used','pool_waiting':'lab_db_pool_waiting',
'capacity':'lab_worker_capacity','busy':'lab_worker_busy','cpu':'lab_container_cpu_cores','memory':'lab_container_memory_bytes'}
output={}
for key,expr in queries.items():
rows=wait_for('metric '+key,lambda:prom(expr),bool,seconds=45,interval=2)
if key in ('p50','p95','p99'): check(math.isfinite(float(rows[0]['value'][1])),'percentile has no finite datapoints; repeat after active load')
output[key]={'query':expr,'result':rows}
forbidden={'user_id','request_id','chat_id','message_id','attachment_id','filename','job_id','trace_id','span_id'}
for item in output.values():
check(all(not forbidden.intersection(r.get('metric',{})) for r in item['result']),'high-cardinality metric labels')
print('PASS metrics snapshot: all expected series present, no forbidden ID labels')
return output
def history(c):
alice,bob,_,chat=c.bootstrap()
att,raw,tid,rid=upload(c,alice,chat)
verify_ready(c,bob,att,raw,seconds=150)
wait_for('stored trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=90)
wait_for('stored API log',lambda:logs(tid,'messenger-api',rid),bool,seconds=90)
time.sleep(6)
historic_time=time.time()
expr='sum(lab_http_requests_total{route=~"/api/.*"})'
def historic():
rows=get(PROM+'/api/v1/query',{'query':expr,'time':str(historic_time)})['data']['result']
return float(rows[0]['value'][1]) if rows else None
before=historic();check(before is not None and before>0,'no historic metric')
compose('restart','--timeout','20','prometheus','loki','tempo',timeout=180)
ready()
wait_for('historical trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=100)
wait_for('historical log',lambda:logs(tid,'messenger-api',rid),bool,seconds=100)
after=historic();check(after==before,'historic metric changed or disappeared')
print('PASS history: saved metrics, logs and traces survive backend restart')
return {'trace_id':tid,'historical_query_timestamp':historic_time,'metric_before':before,'metric_after':after}
def main():
p=argparse.ArgumentParser();p.add_argument('--scenario',choices=['trace-retry','queue-lag','db-error','backend-down','metrics','history'],required=True);args=p.parse_args()
check(os.getenv('LAB6_ALLOW_FAILURE_TEST')=='yes','Set LAB6_ALLOW_FAILURE_TEST=yes on isolated synthetic stack')
ready();c=Client(timeout=12)
fun={'trace-retry':trace_retry,'queue-lag':queue_lag,'db-error':db_error,'backend-down':backend_down,'metrics':metrics_snapshot,'history':history}[args.scenario]
result=fun(c)
file=ROOT/'evidence'/('lab6-'+args.scenario+'.json');file.write_text(json.dumps({'scenario':args.scenario,'status':'PASS','result':result},indent=2,ensure_ascii=False)+'\n',encoding='utf-8')
print('Saved',file.relative_to(ROOT))
if __name__=='__main__':run(main)
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""Fast non-Docker checks for input contracts and cursor scoping."""
import sys
from pathlib import Path
from datetime import datetime, timezone
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.core import make_cursor, read_cursor, parse_limit, APIError
from app.models import CreateUser, CreateChat, CreateMessage
from pydantic import ValidationError
def reject(fn):
try:
fn()
except (APIError, ValidationError, ValueError):
return
raise AssertionError("Expected invalid input")
def main():
token = str(uuid.uuid4())
dt = datetime.now(timezone.utc)
cursor = make_cursor("messages", token, dt, str(uuid.uuid4()))
assert read_cursor(cursor, "messages", token)[0] == dt
reject(lambda: read_cursor(cursor, "messages", str(uuid.uuid4())))
reject(lambda: read_cursor(cursor, "chats", token))
assert parse_limit(None) == 50
reject(lambda: parse_limit("0"))
reject(lambda: parse_limit("101"))
reject(lambda: CreateUser(username="user_name", display_name=" \t", password="Long-enough-secret"))
reject(lambda: CreateUser(username="user_name", display_name="Test", password=" " * 3))
assert CreateMessage(text="A\u0000B").text == "A\u0000B"
reject(lambda: CreateMessage(text=" \t"))
reject(lambda: CreateChat(title="title", member_ids=[token] * 2))
reject(lambda: CreateChat(title="title", member_ids=[token] * 101))
print("PASS: models, cursor scoping, limits, Unicode U+0000")
if __name__ == "__main__":
main()
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python3
"""Additional *live* Lab5 fault tests; operator-controlled, synthetic records only.
broker-persistence: require a publisher-confirmed RabbitMQ message to survive
broker restart without a running dispatcher or worker. This differs from the
outbox-restoration tests where dispatcher publishes *after* broker recovery.
failed-replay: safely inject a deterministic checksum mismatch into one fresh
synthetic attachment's DB metadata, verify permanent failure, restore the
original hash, replay via the operator-only CLI, and verify success plus audit.
staging-cleanup: verify the staged S3 object is absent after ready (with a
matching cleared staging_key in the DB).
No volumes are deleted. The test may stop/restart worker/dispatcher/broker;
run only against a disposable LAB5 environment and never concurrently.
"""
import argparse
import hashlib
import os
import re
import subprocess
import time
import uuid
from client import Client, Failure, check, run
from pipeline_resilience import verify_ready, wait_api
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, expected=(0,), timeout=240):
proc = subprocess.run(
['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout,
)
if proc.returncode not in expected:
# Don't print env vars, tokens or command stdout/stderr by default.
raise Failure('docker compose %s failed (exit %d); inspect ps/logs' %
(' '.join(args[:2]), proc.returncode))
return proc.stdout.strip()
def pg(sql):
"""Read-only or deliberately scoped fault SQL; UUIDs never user-supplied here."""
user = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_USER').strip()
dbname = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_DB').strip()
return compose('exec', '-T', 'postgres', 'psql', '-X', '-q', '-A', '-t',
'-v', 'ON_ERROR_STOP=1', '-U', user, '-d', dbname, '-c', sql)
def scalar(sql):
return pg(sql).splitlines()[-1].strip()
def wait_for(label, action, predicate, seconds=30, interval=0.5):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
last = action()
if predicate(last):
return last
except Failure:
pass
time.sleep(interval)
raise Failure('timed out waiting for ' + label + ', last=' + str(last)[:100])
def job_query(att_id, fields):
uuid.UUID(att_id)
return scalar('SELECT %s FROM processing_jobs WHERE attachment_id = \'%s\'' %
(fields, att_id))
def queue_ready():
out = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', 'list_queues',
'name', 'messages_ready', 'messages_unacknowledged')
for line in out.splitlines():
parts = re.split(r'\s+', line.strip())
if parts[0] == 'attachment-processing.v1' and len(parts) >= 3:
return int(parts[1]), int(parts[2])
raise Failure('RabbitMQ durable queue missing')
def make_upload(c):
alice, bob, _eve, chat = c.bootstrap()
att, original = c.attachment(alice, chat)
check(att['status'] == 'queued', 'upload must return 202 queued')
uuid.UUID(att['id'])
return alice, bob, att, original
def wait_staging_cleared(att_id, seconds=20):
return wait_for('staging_key cleared after ready',
lambda: scalar("SELECT COALESCE(staging_key,'<NULL>') FROM attachments WHERE id='%s'" % att_id),
lambda value: value == '<NULL>', seconds=seconds)
def verify_staging_absent(att_id):
# This executes *inside* the worker container with its normal S3 credentials.
# Never expose S3 secrets to the host or write them to evidence files.
code = '''import sys
from botocore.exceptions import ClientError
from app.attachments import s3_client, BUCKET
key='staging/'+sys.argv[1]+'/original'
try:
s3_client().head_object(Bucket=BUCKET,Key=key)
except ClientError as exc:
if exc.response.get('ResponseMetadata',{}).get('HTTPStatusCode')==404:
print('ABSENT')
else:
raise
else:
print('STILL_PRESENT')'''
result = compose('exec', '-T', 'worker', 'python', '-c', code, att_id)
check(result == 'ABSENT', 'staging S3 object still present: ' + result[:70])
def broker_persistence(c):
compose('stop', 'worker')
compose('up', '-d', 'dispatcher')
baseline = queue_ready()[0]
alice, _bob, att, original = make_upload(c)
print('202 queued:', att['id'], flush=True)
sql = "SELECT COALESCE(bool_and(published_at IS NOT NULL),false) FROM job_outbox WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % att['id']
wait_for('outbox publisher confirm', lambda: scalar(sql), lambda v: v == 't', seconds=30)
wait_for('message in RabbitMQ ready queue', lambda: queue_ready()[0],
lambda n: n >= baseline + 1, seconds=15)
compose('stop', 'dispatcher')
before = queue_ready()[0]
check(before >= baseline + 1, 'published message disappeared before restart')
print('Publisher confirmed; dispatcher and worker stopped; broker queue ready=%d' % before, flush=True)
compose('restart', 'broker')
compose('up', '-d', '--wait', 'broker')
after = wait_for('RabbitMQ queue restoration', lambda: queue_ready()[0],
lambda n: n >= before, seconds=40, interval=1)
check(scalar(sql) == 't', 'outbox publish confirmation lost')
print('Broker restarted; ready messages persisted=%d' % after, flush=True)
# CRITICAL: do NOT restart dispatcher yet; only the restored RabbitMQ message
# can deliver this job to the worker.
compose('up', '-d', 'worker')
verify_ready(c, alice, att, original, seconds=180)
check(job_query(att['id'], 'status') == 'ready', 'job not committed ready')
print('PASS: broker-persistence, confirmed RabbitMQ message survives broker restart without outbox republish')
def failed_replay(c):
compose('stop', 'worker', 'dispatcher')
alice, _bob, att, original = make_upload(c)
ident = att['id']
original_sha = hashlib.sha256(original).hexdigest()
check(original_sha == att['sha256'], 'fixture hash mismatch')
check(scalar("SELECT sha256 FROM attachments WHERE id='%s'" % ident) == original_sha,
'DB hash unexpected before controlled fault')
fault_sha = '0' * 64 if original_sha != '0' * 64 else 'f' * 64
changed = False
try:
updated = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='queued' RETURNING id" %
(fault_sha, ident))
check(ident in updated, 'failed to inject synthetic checksum mismatch')
changed = True
compose('up', '-d', 'dispatcher', 'worker')
def get_state():
return c.call('GET', '/api/v1/attachments/' + ident, actor=alice, schema='Attachment')['json']
failed = wait_for('permanent failed status', get_state,
lambda a: a['status'] == 'failed', seconds=90)
check(failed['error_code'] == 'invalid_image',
'expected permanent invalid_image, got ' + str(failed['error_code']))
check(job_query(ident, 'status') == 'failed', 'processing job must fail')
check(job_query(ident, 'attempt') == '1', 'permanent error must not retry')
check(scalar("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % ident) == '0',
'failed job created variants')
print('Permanent invalid_image confirmed; no retry and no variant rows', flush=True)
# Repair only the generated synthetic row before explicit operator replay.
restored = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='failed' RETURNING id" %
(original_sha, ident))
check(ident in restored, 'synthetic metadata could not be restored')
changed = False
compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident)
verify_ready(c, alice, att, original, seconds=180)
audit = scalar("SELECT count(*) FROM job_replay_audit WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % ident)
check(audit == '1', 'operator replay audit count incorrect: ' + audit)
check(job_query(ident, 'status') == 'ready', 'replayed job not ready')
attempt = job_query(ident, 'attempt')
check(attempt == '1', 'replay did not reset attempt: ' + attempt)
invalid = compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs',
'replay', '--id', ident, expected=(1,))
print('PASS: failed-replay; permanent failure → metadata repaired → replay → ready; audit=1; replay of ready rejected')
finally:
if changed:
# Never leave the synthetic attachment with altered metadata.
pg("UPDATE attachments SET sha256='%s' WHERE id='%s'" % (original_sha, ident))
def staging_cleanup(c):
compose('up', '-d', 'dispatcher', 'worker')
alice, _bob, att, original = make_upload(c)
verify_ready(c, alice, att, original, seconds=180)
wait_staging_cleared(att['id'], seconds=30)
verify_staging_absent(att['id'])
print('PASS: staging-cleanup; ready, DB staging_key=NULL, original staging object absent in S3')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--scenario', required=True,
choices=['broker-persistence', 'failed-replay', 'staging-cleanup'])
args = parser.parse_args()
if os.getenv('LAB5_ALLOW_FAILURE_TEST') != 'yes':
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes; run only on your isolated Lab5 stack')
c = Client('http://localhost:8080')
try:
wait_api()
if args.scenario == 'broker-persistence':
broker_persistence(c)
elif args.scenario == 'failed-replay':
failed_replay(c)
else:
staging_cleanup(c)
finally:
# Restore normal services even when an assertion fails. NO volume deletion.
try:
compose('up', '-d', '--wait', 'broker')
compose('up', '-d', 'dispatcher', 'worker')
except Failure:
print('WARNING: inspect docker compose ps and restore dispatcher/worker manually')
if __name__ == '__main__':
run(main)
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env python3
"""Explicit operator evidence for Lab5 queue durability and failure windows.
Runs only on an isolated lab5 Compose stack. Never resets named volumes. The test
preserves user/session secrets strictly in process memory and prints no tokens.
"""
import argparse
import hashlib
import os
import subprocess
import sys
import time
from pathlib import Path
from client import Client, Failure, check, run
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, expected=(0,)):
p = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=240)
if p.returncode not in expected:
raise Failure('docker compose command failed: %s (exit %s); inspect docker compose ps/logs'
% (' '.join(args[:2]), p.returncode))
return p
def wait_api(timeout=60):
import urllib.request
deadline=time.monotonic()+timeout
while time.monotonic()<deadline:
try:
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2) as r:
if r.status==200:
return
except Exception:
pass
time.sleep(.5)
raise Failure('API did not become ready')
def make_queued(client):
alice,bob,eve,chat=client.bootstrap()
att,original=client.attachment(alice,chat)
check(att['status']=='queued','202 must return queued snapshot')
c=client.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=alice)
attached=client.message(alice,chat,'',attachments=[att['id']])['json']
check(attached['attachment_ids']==[att['id']],'queued attachment cannot be linked')
return alice,bob,att,original
def verify_ready(client,actor,att,original,seconds=180):
r=client.wait_ready(actor,att,wait_seconds=seconds)
check(len(r['variants']) == 3, 'expected exactly three variants, no duplicate entries')
variants={v['name']:v for v in r['variants']}
check(set(variants)=={'thumbnail','crop','optimized'},'missing/extra variants')
check((variants['thumbnail']['width'],variants['thumbnail']['height'])==(256,160), 'thumbnail dimensions')
check((variants['crop']['width'],variants['crop']['height'])==(128,128), 'crop dimensions')
raw=client.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=actor)
check(raw['bytes']==original,'original changed across asynchronous pipeline')
for name in ('thumbnail','crop','optimized'):
v=variants[name]
response=client.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+name,actor=actor)
check(len(response['bytes'])==v['size_bytes'] and hashlib.sha256(response['bytes']).hexdigest()==v['sha256'],
'variant checksum mismatch: '+name)
return r
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--scenario',choices=['paused','outbox-crash','worker-crash'],required=True)
args=parser.parse_args()
if os.getenv('LAB5_ALLOW_FAILURE_TEST')!='yes':
raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes in PowerShell to run destructive failure test')
c=Client('http://localhost:8080')
# All tests make a fresh attachment while dispatcher/worker are stopped.
compose('stop','worker','dispatcher')
try:
alice,bob,att,original=make_queued(c)
print('Created durable queued attachment',att['id'],flush=True)
if args.scenario=='paused':
compose('restart','api1','api2','broker')
compose('up','-d','--wait','broker')
wait_api()
elif args.scenario=='outbox-crash':
# Transaction committed, dispatcher crashes before publish.
compose('run','--rm','-e','LAB5_FAULT_BEFORE_PUBLISH=once',
'dispatcher',expected=(78,))
print('Injected dispatcher exit 78 after committed outbox, before publish')
else:
compose('up','-d','dispatcher')
# Fault after all S3 writes and before PostgreSQL result COMMIT and ack.
compose('run','--rm','-e','LAB5_FAULT_AFTER_S3=once',
'worker',expected=(77,))
print('Injected worker exit 77 after S3 effects, before result/ack')
compose('up','-d','dispatcher','worker')
wait_api()
ready=verify_ready(c,bob,att,original)
check(ready['status']=='ready','attachment not ready')
print('PASS:',args.scenario,'durable queued->ready; original/3 variants verified; no duplicates')
finally:
# Restore the normal topology even after an assertion failure.
compose('up','-d','dispatcher','worker')
if __name__=='__main__':
run(main)
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Real Python OTLP SDK wire-format test against a local in-process receiver.
Requires requirements.txt (does not require Docker or external services).
This is not a Collector/backend integration test.
"""
import os
import sys
import time
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
from opentelemetry.trace import SpanKind
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest
from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest
from app import telemetry as ot
received={k:[] for k in ('/v1/traces','/v1/logs','/v1/metrics')}
class Receiver(BaseHTTPRequestHandler):
def do_POST(self):
data=self.rfile.read(int(self.headers['Content-Length']))
received[self.path].append(data)
self.send_response(200);self.send_header('Content-Type','application/x-protobuf');self.send_header('Content-Length','0');self.end_headers()
def log_message(self,*args): pass
def main():
server=ThreadingHTTPServer(('127.0.0.1',0),Receiver)
threading.Thread(target=server.serve_forever,daemon=True).start()
os.environ['OTEL_ENABLED']='yes'
os.environ['OTEL_EXPORTER_OTLP_ENDPOINT']='http://127.0.0.1:'+str(server.server_port)
ot.init('worker')
with ot.span('POST /api/v1/chats/{chat_id}/attachments',kind=SpanKind.SERVER) as root:
stored=ot.inject(); trace_id=ot.trace_id()
with ot.span('rabbitmq.publish',kind=SpanKind.PRODUCER,context=ot.extract(stored)) as producer:
sent=ot.inject()
with ot.span('rabbitmq.consume',kind=SpanKind.CONSUMER,context=ot.extract(sent)) as consumer:
ot.event('job.ready',job_id='synthetic-id',attempt=2)
ot.count('lab_jobs',{'job_type':'attachment.process.v1','outcome':'ready'})
ot.observe('lab_job_duration',2.5,{'job_type':'attachment.process.v1','outcome':'ready'})
ot.observe('lab_ready_duration',5.0,{'job_type':'attachment.process.v1'})
class FakeS3:
def put_object(self,**kw): return True
ot.S3Tracing(FakeS3()).put_object(Key='SECRET_FILENAME_SENTINEL',Body=b'SECRET_CONTENT_SENTINEL')
ot.flush()
for provider in reversed(ot.providers): provider.shutdown()
server.shutdown()
spans=[s for raw in received['/v1/traces'] for r in ExportTraceServiceRequest.FromString(raw).resource_spans for scope in r.scope_spans for s in scope.spans]
byname={s.name:s for s in spans}
assert byname['rabbitmq.publish'].parent_span_id==root.get_span_context().span_id.to_bytes(8,'big')
assert byname['rabbitmq.consume'].parent_span_id==producer.get_span_context().span_id.to_bytes(8,'big')
assert all(s.trace_id.hex()==trace_id for s in spans)
assert 's3.put_object' in byname
logs=[l for raw in received['/v1/logs'] for r in ExportLogsServiceRequest.FromString(raw).resource_logs for scope in r.scope_logs for l in scope.log_records]
assert any(l.trace_id.hex()==trace_id and l.span_id==consumer.get_span_context().span_id.to_bytes(8,'big') and l.body.string_value=='job.ready' for l in logs)
points=[m for raw in received['/v1/metrics'] for r in ExportMetricsServiceRequest.FromString(raw).resource_metrics for scope in r.scope_metrics for m in scope.metrics]
assert any(m.name=='lab_jobs' and m.sum.data_points[0].as_int==1 for m in points)
assert any(m.name=='lab_job_duration' and list(m.histogram.data_points[0].explicit_bounds)==list(ot.BUCKETS) for m in points)
for raw in sum(received.values(),[]): assert b'SECRET_FILENAME_SENTINEL' not in raw and b'SECRET_CONTENT_SENTINEL' not in raw
print('PASS: real OTLP trace/log/metric encoding, parent links, histogram buckets, safe S3 attributes')
if __name__=='__main__': main()