diff --git a/.env.example b/.env.example index 2b32655..eb0ac2c 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,28 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Copy to .env and REPLACE all sample secrets with random unique values. +# For local synthetic data only. Never commit .env. +POSTGRES_DB=messenger +POSTGRES_USER=lab +POSTGRES_PASSWORD=REPLACE_WITH_RANDOM_LOCAL_PASSWORD SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 +DB_POOL_MAX=8 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +S3_ADMIN_ACCESS_KEY=adminlab4 +S3_ADMIN_SECRET_KEY=REPLACE_WITH_DIFFERENT_RANDOM_ADMIN_SECRET +S3_ACCESS_KEY=messengerapi +S3_SECRET_KEY=REPLACE_WITH_DIFFERENT_RANDOM_API_SECRET + +# RabbitMQ persistent queue (different from session Redis). +BROKER_USER=labworker +BROKER_PASSWORD=REPLACE_WITH_RANDOM_LOCAL_BROKER_PASSWORD JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +JOB_BACKLOG_LIMIT=1000 +WORKER_PREFETCH=1 +# Only for deliberate crash experiment: once. Keep off for normal operation. +LAB5_FAULT_AFTER_S3=off + +# Observability (loopback-only Grafana / diagnostics). No public OTLP ports. +GRAFANA_ADMIN_PASSWORD=REPLACE_WITH_DIFFERENT_RANDOM_GRAFANA_PASSWORD +OTEL_ENABLED=yes +LAB6_FAULT_RETRY_ONCE=off diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..31979f1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.10-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /srv +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && groupadd --gid 999 app && useradd -m --uid 999 --gid app app +COPY app ./app +COPY db ./db +USER app +EXPOSE 8080 +CMD ["python", "-m", "app"] diff --git a/Makefile b/Makefile index 5d86d17..07e73d2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PYTHON ?= python3 +PYTHON ?= python BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= diff --git a/REPORT.md b/REPORT.md index b97ae42..c3593d8 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,208 @@ -# Отчёт — лабораторная 6 +# Отчёт — лабораторная 6. Наблюдаемость -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор / группа: Авзалов Артур Марсович / БПИ2301. +- Целевая оценка: 4; трек 5 не заявлен. +- Статус: кандидат на 4. Реализация проверена на Windows/Docker Desktop 2026-11-17: все шесть сценариев наблюдаемости, публичный smoke, ряд регрессионных тестов Lab5 и визуализация Grafana прошли. Целевая оценка 4; окончательную оценку определяет преподаватель. +- Commit/PR: указывается после публикации. +- ИИ: ChatGPT помог проектировать, реализовывать инструментацию, инфраструктуру, тесты и документацию. Результаты ниже получены при запуске именно Lab6; ChatGPT помог исправить диагностический тест backend-down. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Docker Desktop (Linux containers, Compose V2), Python 3.9+ для публичных HTTP-тестов; Python 3.12 рекомендуется для локальных SDK/unit-проверок. Приложение использует Python 3.12.10. Желательно выделить Docker 6–8 GiB RAM с запасом. Перед запуском остановить Lab5 и другие стенды, занимающие 8080/3000/9090/3100/3200. Не удалять их volumes. -## Архитектура и решения +PowerShell (корень проекта): -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```powershell +Copy-Item .env.example .env +notepad .env +$env:PYTHONUTF8 = "1" +# Заменить ВСЕ REPLACE_WITH_... на разные случайные значения. +# Для паролей в DSN выбрать hex/URL-safe значения, не вставлять секреты в отчёт. +# OTEL_ENABLED=yes, LAB5_FAULT_AFTER_S3=off, LAB6_FAULT_RETRY_ONCE=off. +docker compose --env-file .env config --quiet +docker compose --env-file .env run --rm --no-deps otel-collector validate --config=/etc/otelcol/config.yaml +docker compose --env-file .env run --rm --no-deps --entrypoint /bin/promtool prometheus check config /etc/prometheus/prometheus.yml +docker compose --env-file .env up -d --build --wait +docker compose --env-file .env ps +python scripts/wait_ready.py +make check +make test +``` + +`--wait` для backend без healthcheck подтверждает только запущенный контейнер, не готовность ingestion. Дождаться `/-/ready` Prometheus, `/ready` Loki/Tempo, `/api/health` Grafana, `up` для Collector; расширенные тесты делают это автоматически. Grafana: http://127.0.0.1:3000, пользователь `admin`, пароль `GRAFANA_ADMIN_PASSWORD` из локального `.env` (задаётся при первом создании volume). Dashboard MTUCI → Lab6 | Messenger observability (`/d/lab6-messenger/`). Prometheus alerts: http://127.0.0.1:9090/alerts. Порты диагностики доступны только на loopback; никакого наружного OTLP или прямых API-реплик. + +Для локальных unit/SDK-проверок в отдельном Python 3.12 venv: + +```powershell +python -m pip install -r requirements.txt +python scripts/check_lab6.py +python tests/jobs_sql_contract.py +python tests/own_unit.py +python tests/telemetry_unit.py +# Необязательная полная OpenAPI-валидация (отдельный dev requirement): +python -m pip install -r requirements-dev.txt +make validate +``` + +Безопасная остановка: `docker compose --env-file .env down` (named volumes сохраняются). Только намеренный сброс СВОЕГО учебного стенда: `docker compose --env-file .env down -v` (удаляет и бизнес-данные, и историю телеметрии, не использовать для тестов). `.env`, реальные credentials и данные вне Git. Инициализация `migrate` транзакционна и идемпотентна; `s3-policy` готовит приватные credentials/policy, `s3-bootstrap` bucket, `telemetry-init` устанавливает права директорий named volumes. RabbitMQ queue создаётся приложением с durable/publisher confirms. Provisioning Grafana полностью из Git. + +## Архитектура и перенос Lab5 + +```text +HTTP -> HAProxy -> API1/API2 -> PostgreSQL + Redis + SeaweedFS + -> outbox -> dispatcher -> RabbitMQ -> worker + worker -> S3 + DB ready +API / dispatcher / worker (включая дочерний image process) + | OTLP HTTP/protobuf :4318, внутри telemetry + v + OTel Collector + | metrics /metrics :8889 <- Prometheus scrape + | logs native OTLP HTTP -> Loki + | traces OTLP gRPC -> Tempo + v + Grafana (запросы к трём независимым stores) +``` + +Бизнес-контракт Lab6 и исходные публичные тесты не изменены; добавляется необязательный диагностический response header `X-Trace-Id` на business HTTP при активной телеметрии. Метаданные, состояние job/outbox и идемпотентность живут в PostgreSQL; сессии в Redis AOF; оригиналы/варианты в S3; RabbitMQ доставляет задачи at-least-once. SDK и Collector не участвуют в транзакциях приложения. Worker сохраняет исходный timeout/lease и отдельный process для Pillow; таймаут/ошибки приводят к durable retry через Outbox. Изменения в `app/jobs.py`, `app/main.py`, `app/storage.py`, `app/attachments.py`, `app/image_worker.py` касаются инструментации, измерения backlog и обработки DB statement timeout; dispatcher обслуживает heartbeat даже в idle. Миграции 001–004 сохранены. Новых бизнес-таблиц не нужно: контракт job допускает optional `traceparent`, `tracestate`, они сохраняются в JSONB outbox. + +### Роли / версии / данные / доступ + +| Service | Версия (образ) | Данные, сеть, readiness | +| --- | --- | --- | +| api1/api2 | Python 3.12.10-slim-bookworm, FastAPI 0.115.12 | stateless, frontend/storage/telemetry; `/health/ready`, не опубликованы напрямую | +| lb | HAProxy 3.1.7-alpine | frontend, 127.0.0.1:8080; readiness upstream | +| postgres | 16.10-bookworm | postgres_data, storage, pg_isready | +| redis | 7.4.5-alpine | redis_data, storage, redis-cli ping, AOF | +| s3 | SeaweedFS 4.53 | s3_data, s3_config, storage; bootstrap + API head_bucket | +| broker | RabbitMQ 4.1.8-management-alpine | rabbitmq_data, storage, diagnostics ping | +| dispatcher/worker | Python 3.12.10, Pika 1.3.2, Pillow 11.3.0 | storage/telemetry, restart policy; live fault tests | +| otel-collector | contrib 0.123.0 | memory-only bounded batches, telemetry; scrape :8888/:8889 | +| prometheus | v3.2.1 | prometheus_data, telemetry, loopback 9090; /-/ready | +| loki | 3.4.2 | loki_data, TSDB v13 filesystem, loopback 3100; /ready | +| tempo | 2.7.2 | tempo_data, local blocks + WAL, loopback 3200; /ready | +| grafana | 11.6.0 | grafana_data + Git provisioning, loopback 3000; /api/health | +| telemetry-init | BusyBox 1.37.0 | one-shot chown только корней named volumes, сеть отсутствует | + +API 384 MiB/0.75 CPU на реплику, worker 768 MiB/1 CPU, dispatcher 192 MiB/0.5 CPU, Collector 256 MiB/0.75 CPU, Prometheus 384 MiB, Loki/Tempo 512 MiB каждый, Grafana 384 MiB. Реальные CPU/RAM замеряются на стенде; лимиты не измерения. Prometheus retention 72h/512MB, Loki и Tempo retention 72h (асинхронная очистка backend, не жёсткий disk quota). Все конфиги `infra/otel/` монтируются read-only. `.env.example` содержит логические настройки runtime, Compose отображает POSTGRES_* в DATABASE_URL, Redis URL фиксируется внутренней сетью, S3_* в credentials, BROKER_* в AMQP credentials. Локальный файл `.env` не коммитится; секреты хранятся в окружении контейнеров учебного стенда, не production secret manager. `STAGING_RETENTION_SECONDS` при отсутствии настройки приложения равен 86400. `WORKER_PREFETCH=1`, один активный job на worker. `OTEL_ENABLED=yes` включает SDK; endpoint `http://otel-collector:4318` задаётся Compose. `GRAFANA_ADMIN_PASSWORD` только для локального интерфейса. `LAB6_FAULT_RETRY_ONCE` — операторская настройка, по умолчанию off (не API). + +## Сигналы, запросы и единицы + +SDK OTel API/SDK/OTLP HTTP 1.42.1, явная инструментация `app/telemetry.py`. Стабильные локальные names и ограниченные semantic attributes вместо автоматического capture. `service.name=messenger-api|messenger-dispatcher|messenger-worker`, `service.version=1.6.0`, `service.instance.id=api1|api2|container hostname`, `deployment.environment.name=lab`; Prometheus exporter переводит точки в подчёркивания, добавляет суффиксы секунд/байт/total. Все histogram buckets одинаковы на репликах: 0.005…600 s. Health измеряется, но исключён из business SLI и трассировки (чтобы probe не загрязнял трассы). HTTP duration измеряется до получения заголовков ответа; потоковая передача content целиком не измеряется. Не именуем p99 средним. + +| Сигнал / Prometheus имя | Выражение / единица | +| --- | --- | +| HTTP throughput и 5xx | `sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))`, фильтр `status_class="5xx"` для ошибок; req/s | +| p50/p95/p99 | `histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))`, аналогично 0.5/0.99; секунды | +| Job outcomes | `sum by(outcome)(rate(lab_jobs_total[30s]))`, attempts/s; ready/retry/failed/stale/requeue; hard kill до метрики не считается завершённым attempt | +| Worker processing | `histogram_quantile(0.95,sum by(le)(rate(lab_job_duration_seconds_bucket[30s])))`, секунды claimed attempt | +| Time to ready | `histogram_quantile(0.95,sum by(le)(rate(lab_ready_duration_seconds_bucket[30s])))`, секунды от `processing_jobs.created_at` (время транзакции durable acceptance, близкое, но не точно равное моменту COMMIT) до commit ready, включает retry/backlog | +| Durable backlog | `max(lab_queue_depth)`: число jobs в статусе queued, включая неопубликованные и delayed retry | +| Broker queue | `max(lab_broker_ready)`: ready messages RabbitMQ через passive declare; `lab_broker_consumers` — число consumers; не включает unacked и не равно DB backlog | +| Oldest age | `max(lab_queue_oldest_age_seconds)`: now−created_at старейшей queued job; секунды; нет queued = 0 | +| DB pool | `lab_db_pool_used`, `lab_db_pool_waiting`, `lab_db_pool_capacity` по service_instance_id; connections/requests; used = size−available (включая reserved/создаваемые соединения) | +| Worker saturation | `sum(lab_worker_busy)/clamp_min(sum(lab_worker_capacity),1)`; ratio, active/capacity. Один job на процесс. При отсутствии worker ряд исчезает по expiration, это не доказательство idle | +| CPU/RAM | `lab_container_cpu_cores` (CPU seconds / wall seconds), `lab_container_memory_bytes` (cgroup v2 usage, fallback v1 CPU / RSS дерева процессов) | +| Collector internal | `otelcol_exporter_queue_size`, `otelcol_exporter_queue_capacity`, `otelcol_exporter_send_failed_spans[_total]`, `otelcol_exporter_send_failed_log_records[_total]`, `otelcol_process_memory_rss` / `otelcol_process_memory_rss_bytes`; учитывать текущую нормализацию выбранной версии | + +`method`, route template, `status_class`, `job_type`, `outcome`, `queue` — ограниченный набор label values, `service_instance_id` ограничен числом экземпляров за retention; нет raw URL, filename, user/chat/message/attachment/request/job ID. Метрики cumulative, Collector выставляет Prometheus exposition. `metric_expiration: 15s` ограничивает stale gauges (нет данных при недоступной БД, а не искусственный 0). Percentiles строятся по сумме histogram buckets всех реплик (не усреднение p95). При отсутствии событий rate/quantile может быть NaN: требуется свежая нагрузка, NaN не заменяем нулём искусственно. + +### Сквозная трасса и логи + +1. API создаёт реальный SERVER span на business route; `db.transaction` и `s3.upload_fileobj` видны как dependency spans. Request ID остаётся прежним; `http.completed` содержит request_id, status_code, method, route, trace/span ID, timestamp и severity. +2. Во время транзакции job/outbox `job_payload()` добавляет W3C `traceparent`/`tracestate`. Dispatcher извлекает их; создаёт PRODUCER span и внедряет **свой** context в сообщение RabbitMQ. Outbox хранит исходный контекст. На retry вызов из CONSUMER сохраняет его context; lease rescue/replay без активного span использует первый сохранённый context. +3. Worker извлекает context, CONSUMER является child producer; проверяет job/lease в БД. `image.subprocess` передаёт context дочернему процессу в env без секретов телеметрии; видны `image.transform`, `db.load_staging_metadata`, `s3.get_object`, `s3.put_object`, `s3.copy_object`, затем `db.publish_result`. Worker завершает `job.ready` или `job.retry`/`job.failed`, ack после durable результата/retry. В случае hard kill последний span может не успеть экспортироваться; retry продолжает trace. +4. В Grafana Explore → Tempo по `X-Trace-Id` / trace_id; открываем span; Loki: `{service_name="messenger-api"} | trace_id="" | request_id=""`, worker `{service_name="messenger-worker"} | trace_id=""`. Поля `trace_id`, `span_id`, `event` — OTLP structured metadata; тела записей фиксированные технические события. Технический job_id только в trace/log для конкретного учебного задания; SQL text, exception messages/stack traces, raw URLs, Authorization, passwords, session tokens, тело сообщений и файлы не передаются SDK. Отключён uvicorn access log, только отдельный безопасный logger передаётся в OTLP. Это проверкой синтетических sentinel полей подтверждено локально, не исчерпывающий security-аудит. + +### SLI / SLO и alerts + +SLI = 1−(число business HTTP 5xx / число всех завершённых business HTTP responses), health исключён, 4xx клиента не считаются отказом сервера. Окно 5 минут; **SLO >= 99%** (учебный, выбран нами, не обещание SLA). Dashboard SLI: `1 - ((sum(increase(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[5m])) or vector(0)) / sum(increase(lab_http_requests_total{route=~"/api/.*"}[5m])))`. Если нет трафика — нет измеримого SLI. SLI не учитывает отказ между клиентом и HAProxy либо ответы самого HAProxy, поскольку SDK в API; для production нужны LB/probe metrics. Время до ready отдельно измеряется histogram; успешные ready не учитывают навсегда failed, поэтому не подменяем им SLI доступности. + +Правила `infra/otel/prometheus/alerts.yml`: `LabAPIHighErrorRate` >5% business 5xx за 30 s, `for: 9s`; `LabQueueLag` age >15 s и depth>0, `for: 12s`. Это локальные diagnostic alert thresholds, не burn-rate track 5. Prometheus evaluation/scrape 3s; firing/pending/inactive (inactive после firing = resolved) в Prometheus UI/API, Grafana dashboard `ALERTS`. Проверка искусственными отказами — ниже; внешней доставки нет. + +### Backpressure и сбой телеметрии + +SDK async BatchSpan/BatchLogRecordProcessor queue 512 items/64 batch, HTTP exporter 2s timeout; metric reader 3s interval/2s export timeout. Не ждём backend для HTTP/normal commit; дочерний process ограниченно flush'ит (добавляет latency, остаётся внутри JOB_TIMEOUT), потеря телеметрии допустима. Collector memory_limiter 180 MiB (spike 40 MiB), container limit 256 MiB, GOMEMLIMIT 180MiB. Batch 128/256 items, timeout 1s; каждый Tempo/Loki exporter queue 64 batches, один consumer, request timeout 3s, retry до 10s. При переполнении/отказе telemetry может быть потеряна; нет бесконечной durable очереди и нет гарантий delivery telemetry. Prometheus scrape хранит history самостоятельно. `backend-down` проверяет успешные бизнес-записи/обработку изображения при остановленном Tempo, рост failed exports, sampled RSS и queue size, отсутствие OOM/restart и свежий trace после восстановления. Failed export counter отражает неуспешный экспорт Collector (не точное число всех потерянных spans SDK; повторные попытки и обрывы могут влиять). Измерение на коротком сценарии не доказательство бессрочной нагрузки. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Локальные проверки (в среде подготовки, 2026-11-17) см. [evidence/offline_validation.md](evidence/offline_validation.md). Фактические результаты Docker-прогона на Windows 2026-11-17 сведены в [evidence/verified_results_2026-11-17.md](evidence/verified_results_2026-11-17.md). JSON-файлы сформированы тестами на стенде; скриншоты Grafana — в `evidence/grafana/`. Неповторённые проверки отмечены отдельно. Скрипты добавлены отдельно, исходные `tests/smoke.py`, `client.py`, `load.py`, `resilience.py`, fixtures и OpenAPI сохранены byte-for-byte. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда/сценарий | Результат / evidence | +| --- | --- | --- | +| Структура, SQL, SDK | `make check`, `python scripts/check_lab6.py`, `python tests/jobs_sql_contract.py`, `python tests/telemetry_unit.py`, `python tests/own_unit.py` | локально PASS; SDK wire-format test не проверяет Collector/backend | +| Основной API и регрессия Lab5 | `make test`, `attachments_negative`, `idempotency_parallel`, `own_unit`, `pipeline_resilience` (paused/outbox-crash/worker-crash) | PASS на Lab6; отдельные `resilience.py` и `pipeline_extended.py` в данном прогоне не подтверждены | +| Все три stores, dashboard/datasources, метрики + низкая кардинальность | `observability_live.py --scenario metrics` | PASS: ожидаемые ряды и отсутствие запрещённых ID labels; `evidence/lab6-metrics.json` | +| Upload/producer/consumer/retry, лог+trace, S3+DB result | `--scenario trace-retry` | PASS: API/producer/consumer/S3/DB с retry, связанные логи; `evidence/lab6-trace-retry.json` | +| Queue age / depth / SLO & alert | `--scenario queue-lag` | PASS: возраст 27.574387 s; queued=1; firing → ready → resolved; `evidence/lab6-queue-lag.json` | +| DB latency / 5xx alert | `--scenario db-error` | PASS: QueryCanceled/503, error ratio 1.0; firing → resolved; `evidence/lab6-db-error.json` | +| Collector ограничен, telemetry backend down | `--scenario backend-down` | PASS после исправления теста: 16 успешных записей, sampled max RSS 200.43 MiB / 256 MiB, queue 18/64, свежая трасса после восстановления; `evidence/lab6-backend-down.json` | +| Сохранение метрик/логов/traces при restart | `--scenario history` | PASS: сохранены метрики, логи, трассы после перезапуска backend; `evidence/lab6-history.json` | +| Нагрузка, percentiles, ресурсы | `tests/load.py`, `attachments_concurrent.py`, dashboard CPU/RAM, PromQL | PASS: 10/10 HTTP 202 → ready, по 3 варианта; acceptance 3.976 RPS, p50 2296.5 ms, max 2484.0 ms, все ready за 36.406 s; `evidence/attachments-concurrent.json`; 16 панелей Grafana показывают данные. Отдельный `tests/load.py` ещё не подтверждён | -## Эксперименты и отказы +### Команды полного прогона на Windows -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Список включает как уже выполненные, так и дополнительные регрессионные проверки. Фактический статус — в сводке доказательств; команды сами по себе не означают PASS. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +```powershell +$env:PYTHONUTF8 = "1" +$env:LAB5_ALLOW_FAILURE_TEST = "yes" +$env:LAB6_ALLOW_FAILURE_TEST = "yes" +# Все fault tests строго последовательно, только на отдельном учебном стенде. +make test +python tests/attachments_negative.py +python tests/idempotency_parallel.py +python tests/own_unit.py +python tests/pipeline_resilience.py --scenario paused +python tests/pipeline_resilience.py --scenario outbox-crash +python tests/pipeline_resilience.py --scenario worker-crash +python tests/pipeline_extended.py --scenario broker-persistence +python tests/pipeline_extended.py --scenario failed-replay +python tests/pipeline_extended.py --scenario staging-cleanup +python tests/load.py --duration 30 --concurrency 4 --output evidence/load.json +python tests/attachments_concurrent.py --concurrency 10 --output evidence/attachments-concurrent.json +python tests/observability_live.py --scenario metrics +python tests/observability_live.py --scenario trace-retry +python tests/observability_live.py --scenario queue-lag +python tests/observability_live.py --scenario db-error +python tests/observability_live.py --scenario backend-down +python tests/observability_live.py --scenario history +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd +docker compose --env-file .env up -d --no-deps --wait api1 api2 +make test +``` + +На Linux соответствующие hooks `restart.sh`, `stop-one.sh` (executable). Не запускать сценарии одновременно. Fault scripts управляют реальными контейнерами, не удаляют volumes. `trace-retry` временно пересоздаёт worker с operator-only fault once, затем восстанавливает off. `db-error` удерживает lock не более 60s после захвата; освобождение в конце транзакции; не применять на production. `history` перезапускает telemetry stores с named volumes. При ошибке смотреть `docker compose logs --tail=80 `, не публиковать secrets. `make check` не заменяет runtime. Если Docker не стартует, не менять публичные тесты ради PASS. + +## Инциденты и runbook + +**Инцидент 1, гипотеза: остановлен worker.** `queue-lag` принимает синтетический upload при остановленном worker. Через метрики backlog/oldest age определяем растущую очередь, сравниваем broker ready/consumer count; API может продолжать отвечать 202, CPU worker отсутствует, DB pool нормальный. `LabQueueLag` firing; trace upload → publish есть, consume отсутствует. Запустить worker и дождаться ready, depth=0, alert inactive. Время обнаружения/возраст/глубина — в JSON результата. Факт (17.11): `queue-lag` PASS, age 27.574387 s, queued=1 при firing; после запуска worker задание стало ready, alert resolved. См. `evidence/lab6-queue-lag.json` и [графики очереди](evidence/grafana/02-worker-and-queue.png). + +**Инцидент 2, гипотеза: DB заблокирована, не медленный handler.** `db-error` удерживает кратковременный ACCESS EXCLUSIVE LOCK на `users` в отдельной psql транзакции, health SELECT 1 остаётся рабочим. Создаётся несколько business GET `/api/v1/users/me`, которые получают 503 после database statement timeout ~5s или pool timeout. В Grafana: HTTP errors/p95 растут, `db.transaction` занимает основное время и имеет error.type QueryCanceled, связанный http.completed лог с request_id; отличить от долгого обработчика с нормальными dependency spans. `LabAPIHighErrorRate` firing; после COMMIT lock запросы снова 200, error rate падает и alert inactive. Факт (17.11): `db-error` PASS, QueryCanceled и HTTP 503 обнаружены, error ratio=1.0 при firing, после снятия блокировки alert resolved. См. `evidence/lab6-db-error.json` и [графики API](evidence/grafana/01-api-and-jobs.png). + +**Инцидент 3: Tempo недоступна.** `backend-down` останавливает Tempo, посылает synthetic messages и image upload. Prometheus продолжает считать business telemetry; Collector failed spans/queue/RSS показывают сбой экспорта, API/worker продолжают обработку. После возврата Tempo создаётся новая трасса, старые могли потеряться. Факт (17.11): 16 успешных записей, обработка изображения сохранена; RSS Collector до отказа 197.35 MiB, наблюдаемый максимум 200.43 MiB при лимите 256 MiB; очередь выросла с 0/64 до 18/64. Ошибки экспорта наблюдались, свежая трасса после восстановления доступна. См. `evidence/lab6-backend-down.json` и [графики Collector](evidence/grafana/04-collector-alerts-slo.png). Первоначальный тест завершился на проверке RSS; исправлен только тест (поддержка варианта имени метрики с `_bytes`, строгая проверка очереди и диагностика лимитов); последующий живой прогон PASS. + +Для нагрузки фиксировать hardware/выделенные Docker CPU/RAM, прогрев после `up`, дату, commit, размер файла 111817 B, длительность 30s и concurrency 4 для POST messages, concurrency 10 для upload; сохранить `load.json`, `attachments-concurrent.json`, показатели p50/p95/p99 из dashboard, CPU/RAM через PromQL/периодические замеры `docker stats`. Closed-loop нагрузки не доказывают максимальную capacity; p95 из histogram приблизителен. В `evidence/` хранить только обезличенные summary и технические IDs синтетических trace, не raw secrets/requests. Скриншоты Grafana/alerts полезны как дополнение, не вместо stored results. + +### Визуальная проверка Grafana (17.11.2026) + +Все 16 панелей на четырёх приложенных скриншотах содержат данные реального запуска. Снимки показывают часовой интервал с тестовыми инцидентами и последующей нагрузкой; точные значения и состояния берутся из результатов сценариев, визуальные оценки округлены. + +| Доказательство | Что видно | +| --- | --- | +| [API и jobs](evidence/grafana/01-api-and-jobs.png) | Throughput, всплеск доли 5xx до 100%, p95/p99 около 10 s в момент сбоя, job outcomes | +| [Worker и очередь](evidence/grafana/02-worker-and-queue.png) | Processing duration, time to ready (с учётом очереди), backlog, пик возраста около 27 s | +| [Pool и ресурсы](evidence/grafana/03-pool-and-resources.png) | DB pool capacity 8 на реплику, worker saturation достигает 100%, CPU и sampled memory по экземплярам | +| [Collector, alerts, SLO](evidence/grafana/04-collector-alerts-slo.png) | Очередь экспорта Tempo при ограничении 64, failed exports, firing/pending, падение SLI ниже 99% и последующее восстановление | + +Для показа связи лога и трассы в Grafana Explore используйте сохранённый trace ID `4d8711b71965876a59aa5594ad0d51a0` из `trace-retry`, пока данные доступны по retention. При истечении retention повторите сценарий. PNG подтверждают визуализацию метрик; связанные логи и spans подтверждены тестом `trace-retry`, а не отдельными скриншотами Explore. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Один Docker-host и одиночные PG, Redis, S3, RabbitMQ, LB, Collector и telemetry backends. При их потере не заявляем HA. Бизнес-данные и telemetry разделены, at-least-once не exactly-once. Наблюдаемость best-effort: SDK queue/Collector memory pressure, hard crash, offline backend могут приводить к пропускам traces/logs/metrics. Трассируем зависимость до заголовков/вызова S3, не каждую SQL-команду и не весь downloaded stream. No sampling research / burn-rate track. Нагрузочные и fault evidence приведены выше. Замеры ограничены коротким сценарием и 10 одновременными upload; отдельно `tests/load.py`, повторный полный persistence/failover бизнес-стенда и три сценария `pipeline_extended.py` в предоставленном выводе Lab6 отсутствуют. Параметры CPU/RAM Docker-host и полный вывод JSON не предоставлены для этой сводки. Примерные пароли использовались на локальном тестовом стенде; при новом развёртывании заменить их перед запуском, не коммитить `.env`. + +## Источники + +- [Описание задания](README.md), [контракт observability](contracts/observability.md), [схема job](contracts/jobs.schema.json), [карта курса](COURSE.md). +- [OpenTelemetry Collector configuration](https://opentelemetry.io/docs/collector/configuration/), [internal telemetry](https://opentelemetry.io/docs/collector/internal-telemetry/), [context propagation](https://opentelemetry.io/docs/concepts/context-propagation/), [W3C Trace Context](https://www.w3.org/TR/trace-context/). +- [Prometheus histograms](https://prometheus.io/docs/practices/histograms/), [Loki OTLP ingestion](https://grafana.com/docs/loki/latest/send-data/otel/), [Tempo configuration](https://grafana.com/docs/tempo/latest/configuration/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/). diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e12a193 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Lab 3 messenger API.""" diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 0000000..9f5a6ff --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,4 @@ +import os +import uvicorn +if __name__ == "__main__": + uvicorn.run("app.main:app", host="0.0.0.0", port=int(os.getenv("APP_PORT", "8080")), workers=1, access_log=False, timeout_graceful_shutdown=8) diff --git a/app/api.py b/app/api.py new file mode 100644 index 0000000..8f9ad0f --- /dev/null +++ b/app/api.py @@ -0,0 +1,193 @@ +"""Messenger HTTP handlers; state lives only in PostgreSQL/Redis.""" +import hashlib +import json +import re +import uuid + +import psycopg +from argon2 import PasswordHasher +from fastapi import APIRouter, Request + +from .auth import current_user, hasher, login, logout, public_user +from .core import APIError, invalid, iso, make_cursor, parse_limit, parse_uuid, read_cursor +from .models import CreateChat, CreateMessage, CreateUser +from .storage import db + +router = APIRouter() +KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$", re.ASCII) + + +def actor(req: Request) -> dict: + return getattr(req.state, "actor", None) or current_user(req) + + +def chat_detail(conn, chat: dict) -> dict: + members = conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position", (chat["id"],)).fetchall() + return {"id": str(chat["id"]), "title": bytes(chat["title"]).decode("utf-8"), "created_at": iso(chat["created_at"]), "member_ids": [str(x["user_id"]) for x in members]} + + +def message_detail(conn, row: dict) -> dict: + linked = conn.execute("SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", (row["id"],)).fetchall() + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": bytes(row["text"]).decode("utf-8"), "created_at": iso(row["created_at"]), "attachment_ids": [str(x["attachment_id"]) for x in linked]} + + +def require_chat(conn, chat_id: str, actor_id: str) -> dict: + row = conn.execute("SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id WHERE c.id=%s AND cm.user_id=%s", (chat_id, actor_id)).fetchone() + if row is None: + raise APIError(404, "not_found", "Object not found") + return row + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +def ready(): + from .storage import ready as storage_ready + if not storage_ready(): + raise APIError(503, "unavailable", "Service not ready") + return {"status": "ok"} + + +@router.post("/api/v1/users", status_code=201) +def register(body: CreateUser): + hashed = hasher.hash(body.password) + ident = str(uuid.uuid4()) + try: + with db() as conn: + row = conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (ident, body.username, body.display_name.encode("utf-8"), hashed)).fetchone() + return public_user(row) + except psycopg.errors.UniqueViolation: + raise APIError(409, "conflict", "Username already exists") from None + + +@router.post("/api/v1/auth/sessions", status_code=201) +def sessions(req: Request): + return login(req) + + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +def sessions_delete(req: Request): + logout(req) + return None + + +@router.get("/api/v1/users/me") +def users_me(req: Request): + return actor(req) + + +@router.post("/api/v1/chats", status_code=201) +def chats_create(req: Request, body: CreateChat): + ident = actor(req)["id"] + if ident not in body.member_ids: + raise invalid("Current user must belong to the chat") + chat_id = str(uuid.uuid4()) + try: + with db() as conn: + with conn.transaction(): + # Validate all members; FK constraints are the final authority for races. + found = conn.execute("SELECT id FROM users WHERE id = ANY(%s::uuid[])", (body.member_ids,)).fetchall() + if len(found) != len(body.member_ids): + raise invalid("Unknown member") + row = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (chat_id, body.title.encode("utf-8"))).fetchone() + for pos, member in enumerate(body.member_ids): + conn.execute("INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)", (chat_id, member, pos)) + result = chat_detail(conn, row) + return result + except psycopg.errors.ForeignKeyViolation: + raise invalid("Unknown member") from None + + +@router.get("/api/v1/chats") +def chats_list(req: Request, limit: str | None = None, cursor: str | None = None): + who = actor(req)["id"] + n = parse_limit(limit) + mark = read_cursor(cursor, "chats", who) + with db() as conn: + if mark: + existing = conn.execute("SELECT 1 FROM chats c JOIN chat_members cm ON c.id=cm.chat_id WHERE c.id=%s AND c.created_at=%s AND cm.user_id=%s", (mark[1], mark[0], who)).fetchone() + if not existing: + raise invalid("Invalid cursor") + rows = conn.execute(""" + SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id + WHERE cm.user_id=%s AND (%s::timestamptz IS NULL OR (c.created_at,c.id)>(%s::timestamptz,%s::uuid)) + ORDER BY c.created_at,c.id LIMIT %s + """, (who, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall() + items = [chat_detail(conn, row) for row in rows[:n]] + more = len(rows) > n + last = rows[n-1] if more else None + return {"items": items, "next_cursor": make_cursor("chats", who, last["created_at"], str(last["id"])) if last else None} + + +@router.get("/api/v1/chats/{chat_id}") +def chats_get(req: Request, chat_id: str): + who = actor(req)["id"] + ident = parse_uuid(chat_id) + with db() as conn: + row = require_chat(conn, ident, who) + return chat_detail(conn, row) + + +@router.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def messages_create(req: Request, chat_id: str, body: CreateMessage): + who = actor(req)["id"] + ident = parse_uuid(chat_id) + key = req.headers.get("Idempotency-Key") + if not key or not KEY_RE.fullmatch(key): + raise invalid("Invalid Idempotency-Key") + # Lab3 has only text; canonical JSON avoids unstable formatting/field ordering. + fingerprint = hashlib.sha256(json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() + lock_scope = f"{who}:POST:/api/v1/chats/{ident}/messages:{key}" + with db() as conn: + with conn.transaction(): + require_chat(conn, ident, who) + # Transaction-scoped advisory lock is GLOBAL across API replicas. + conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (lock_scope,)) + previous = conn.execute(""" + SELECT idem.body_sha256, m.* FROM message_idempotency idem + JOIN messages m ON m.id=idem.message_id + WHERE idem.actor_id=%s AND idem.chat_id=%s AND idem.method='POST' AND idem.idem_key=%s + """, (who, ident, key)).fetchone() + if previous: + if previous["body_sha256"] != fingerprint: + raise APIError(409, "conflict", "Idempotency key used with another body") + return message_detail(conn, previous) + # 404 for foreign/nonexistent IDs; check BEFORE any message insert. + if body.attachment_ids: + owned = conn.execute("SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])", (ident, body.attachment_ids)).fetchall() + if len(owned) != len(body.attachment_ids): + raise APIError(404, "not_found", "Attachment not found") + if any(item["status"] == "failed" for item in owned): + raise APIError(409, "conflict", "Failed attachment cannot be linked") + message_id = str(uuid.uuid4()) + row = conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (message_id, ident, who, body.text.encode("utf-8"))).fetchone() + for pos, attachment_id in enumerate(body.attachment_ids): + conn.execute("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", (message_id, attachment_id, pos)) + conn.execute("INSERT INTO message_idempotency(actor_id,chat_id,method,idem_key,body_sha256,message_id) VALUES (%s,%s,'POST',%s,%s,%s)", (who, ident, key, fingerprint, message_id)) + return message_detail(conn, row) + + +@router.get("/api/v1/chats/{chat_id}/messages") +def messages_list(req: Request, chat_id: str, limit: str | None = None, cursor: str | None = None): + who = actor(req)["id"] + ident = parse_uuid(chat_id) + n = parse_limit(limit) + mark = read_cursor(cursor, "messages", ident) + with db() as conn: + require_chat(conn, ident, who) + if mark: + existing = conn.execute("SELECT 1 FROM messages WHERE id=%s AND chat_id=%s AND created_at=%s", (mark[1], ident, mark[0])).fetchone() + if not existing: + raise invalid("Invalid cursor") + rows = conn.execute(""" + SELECT * FROM messages WHERE chat_id=%s + AND (%s::timestamptz IS NULL OR (created_at,id)>(%s::timestamptz,%s::uuid)) + ORDER BY created_at,id LIMIT %s + """, (ident, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall() + with db() as conn: + items = [message_detail(conn, row) for row in rows[:n]] + last = rows[n-1] if len(rows)>n else None + return {"items": items, "next_cursor": make_cursor("messages", ident, last["created_at"], str(last["id"])) if last else None} diff --git a/app/attachments.py b/app/attachments.py new file mode 100644 index 0000000..9c2f878 --- /dev/null +++ b/app/attachments.py @@ -0,0 +1,240 @@ +"""Lab5 attachments: synchronous durable admission, asynchronous image pipeline. + +202 means verified bytes were staged to private S3 and metadata + a job + an +outbox event were committed in ONE PostgreSQL transaction. No broker availability +is required for admission; the dispatcher replays unsent intents. +""" +import hashlib +import os +import tempfile +import threading +import uuid +from contextlib import closing + +import boto3 +from boto3.s3.transfer import TransferConfig +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError +from fastapi import APIRouter, Query, Request +from fastapi.responses import StreamingResponse +from PIL import Image, ImageFile, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool + +from .auth import current_user +from .core import APIError, invalid, iso, parse_uuid +from .storage import db +from . import telemetry as ot + +router = APIRouter() +BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +SPOOL_MEMORY_BYTES = 1024 * 1024 +S3_BUFFER_BYTES = 256 * 1024 +DECODE_LIMIT = threading.BoundedSemaphore(1) +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +ImageFile.LOAD_TRUNCATED_IMAGES = False + + +def s3_client(): + return ot.S3Tracing(boto3.client( + "s3", endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name="us-east-1", use_ssl=False, + config=Config(s3={"addressing_style": "path"}, connect_timeout=3, + read_timeout=10, retries={"max_attempts": 1}), + )) + + +def s3_error(exc): + return APIError(503, "unavailable", "Object storage unavailable") + + +def format_attachment(row, variants=()): + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), "filename": row["filename"], + "content_type": row["content_type"], "size_bytes": row["size_bytes"], + "sha256": row["sha256"], "status": row["status"], + "variants": [{k: item[k] for k in ("name", "content_type", "size_bytes", "width", "height", "sha256")} + for item in variants] if row["status"] == "ready" else [], + "error_code": row["error_code"] if row["status"] == "failed" else None, + "created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"]), + } + + +def require_member(conn, chat_id, user_id): + if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone(): + raise APIError(404, "not_found", "Object not found") + + +def attachment_row(conn, attachment_id, user_id): + row = conn.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON cm.chat_id=a.chat_id + WHERE a.id=%s AND cm.user_id=%s + """, (attachment_id, user_id)).fetchone() + if not row: + raise APIError(404, "not_found", "Object not found") + return row + + +def validate_filename(filename: str): + if not (1 <= len(filename) <= 255) or any(c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename): + raise invalid("Invalid filename") + + +def validate_image(fileobj, mime: str): + """Validate MIME, magic, dimensions and complete decode; worker does the resize.""" + try: + fileobj.seek(0) + with Image.open(fileobj) as im: + if im.format not in ("PNG", "JPEG"): + raise APIError(415, "unsupported_media_type", "Unsupported image encoding") + actual = "image/png" if im.format == "PNG" else "image/jpeg" + if actual != mime: + raise APIError(415, "unsupported_media_type", "Content-Type does not match image") + if im.width * im.height > MAX_PIXELS: + raise APIError(413, "payload_too_large", "Image dimensions too large") + im.verify() + fileobj.seek(0) + with DECODE_LIMIT: + with Image.open(fileobj) as im: + im.load() + except APIError: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise APIError(413, "payload_too_large", "Image dimensions too large") from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise APIError(415, "unsupported_media_type", "Invalid image") from None + finally: + fileobj.seek(0) + + +def _admit_upload(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest): + from .jobs import job_payload + stage_key = f"staging/{attachment_id}/original" + key = f"attachments/{attachment_id}/original" + backup = f"recovery/{attachment_id}/original" + job_id = str(uuid.uuid4()) + outbox_id = str(uuid.uuid4()) + client = s3_client() + try: + spool.seek(0) + client.upload_fileobj(spool, BUCKET, stage_key, + ExtraArgs={"ContentType": mime}, + Config=TransferConfig(use_threads=False, multipart_threshold=16*1024*1024, + io_chunksize=S3_BUFFER_BYTES)) + except (BotoCoreError, ClientError, EndpointConnectionError, OSError) as exc: + raise s3_error(exc) from exc + + # Atomic DB metadata + intent. If DB commit is ambiguous, do not delete the + # staged object: an accepted task may already exist; operator cleanup handles it. + with db() as conn: + with conn.transaction(): + require_member(conn, chat_id, user_id) + # Small educational admission bound; pre-existing queued tasks remain safe. + pending = conn.execute("SELECT count(*) AS c FROM processing_jobs WHERE status IN ('queued','processing')").fetchone()["c"] + if pending >= int(os.getenv("JOB_BACKLOG_LIMIT", "1000")): + raise APIError(429, "rate_limited", "Processing queue is full") + row = conn.execute(""" + INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename, + content_type,size_bytes,sha256,status,staging_key,pipeline_version) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,1) RETURNING * + """, (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest, stage_key)).fetchone() + conn.execute(""" + INSERT INTO processing_jobs(id,attachment_id,pipeline_version,status,attempt) + VALUES (%s,%s,1,'queued',1) + """, (job_id, attachment_id)) + payload = job_payload(job_id, attachment_id, 1, 1) + conn.execute(""" + INSERT INTO job_outbox(id,job_id,payload) VALUES (%s,%s,%s::jsonb) + """, (outbox_id, job_id, __import__('json').dumps(payload))) + ot.event("upload.accepted", job_id=job_id) + return format_attachment(row) # snapshot = queued, even if worker is very fast + + +@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)): + from .api import actor, require_chat + who = actor(req)["id"] + chat_id = parse_uuid(chat_id) + validate_filename(filename) + mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if mime not in ("image/png", "image/jpeg"): + raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported") + with db() as conn: + require_chat(conn, chat_id, who) + announced = req.headers.get("content-length") + if announced: + try: + if int(announced) > MAX_BYTES: + raise APIError(413, "payload_too_large", "Image too large") + if int(announced) < 0: + raise invalid("Invalid Content-Length") + except ValueError: + raise invalid("Invalid Content-Length") from None + size = 0 + digest = hashlib.sha256() + with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool: + async for chunk in req.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise APIError(413, "payload_too_large", "Image too large") + digest.update(chunk) + spool.write(chunk) + if not size: + raise APIError(415, "unsupported_media_type", "Empty image") + await run_in_threadpool(validate_image, spool, mime) + return await run_in_threadpool(_admit_upload, spool, attachment_id=str(uuid.uuid4()), + chat_id=chat_id, user_id=who, filename=filename, + mime=mime, size=size, digest=digest.hexdigest()) + + +@router.get("/api/v1/attachments/{attachment_id}") +def get_attachment(req: Request, attachment_id: str): + who = getattr(req.state, "actor", None) or current_user(req) + with db() as conn: + row = attachment_row(conn, parse_uuid(attachment_id), who["id"]) + variants = conn.execute(""" + SELECT name,content_type,size_bytes,width,height,sha256 + FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s + ORDER BY CASE name WHEN 'thumbnail' THEN 0 WHEN 'crop' THEN 1 ELSE 2 END + """, (row["id"], row["pipeline_version"])).fetchall() if row["status"] == "ready" else [] + return format_attachment(row, variants) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment(req: Request, attachment_id: str, variant: str = "original"): + who = getattr(req.state, "actor", None) or current_user(req) + with db() as conn: + row = attachment_row(conn, parse_uuid(attachment_id), who["id"]) + if row["status"] != "ready": + raise APIError(409, "conflict", "Attachment is not ready") + if variant == "original": + key, mime, size = row["object_key"], row["content_type"], row["size_bytes"] + else: + v = conn.execute(""" + SELECT * FROM attachment_variants + WHERE attachment_id=%s AND pipeline_version=%s AND name=%s + """, (row["id"], row["pipeline_version"], variant)).fetchone() + if not v: + raise APIError(404, "not_found", "Object not found") + key, mime, size = v["object_key"], v["content_type"], v["size_bytes"] + client = s3_client() + try: + response = client.get_object(Bucket=BUCKET, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"): + raise APIError(503, "unavailable", "Stored object missing") from exc + raise s3_error(exc) from exc + except BotoCoreError as exc: + raise s3_error(exc) from exc + + def iterator(): + with closing(response["Body"]) as body: + while chunk := body.read(S3_BUFFER_BYTES): + yield chunk + + return StreamingResponse(iterator(), media_type=mime, headers={"Content-Length": str(size)}) diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 0000000..8286fb3 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,104 @@ +"""HTTP Basic only on login; bearer sessions shared by all API replicas.""" +import base64 +import binascii +import hashlib +import re +import secrets +import uuid +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError, VerificationError +from fastapi import Request +from redis.exceptions import RedisError + +from .core import APIError, SESSION_TTL, iso +from .storage import cache, db + +hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +PASSWORD_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +def public_user(row: dict): + return {"id": str(row["id"]), "username": row["username"], "display_name": bytes(row["display_name"]).decode("utf-8"), "created_at": iso(row["created_at"])} + + +def unauthorized() -> APIError: + return APIError(401, "unauthorized", "Unauthorized") + + +def challenge(req: Request) -> str: + return 'Basic realm="messenger", charset="UTF-8"' if req.url.path == "/api/v1/auth/sessions" else 'Bearer realm="messenger"' + + +def bearer_token(req: Request) -> str: + value = req.headers.get("Authorization", "") + if not value.startswith("Bearer ") or len(value) <= 7 or " " in value[7:] or len(value) > 512: + raise unauthorized() + try: + value[7:].encode("ascii") + except UnicodeEncodeError: + raise unauthorized() from None + return value[7:] + + +def session_key(token: str) -> str: + return "session:" + hashlib.sha256(token.encode("ascii")).hexdigest() + + +def current_user(req: Request) -> dict: + token = bearer_token(req) + try: + raw = cache().get(session_key(token)) + except (RedisError, UnicodeError) as exc: + raise APIError(503, "unavailable", "Session store unavailable") from exc + if not raw: + raise unauthorized() + try: + ident = str(uuid.UUID(raw)) + except (ValueError, TypeError): + try: + cache().delete(session_key(token)) + except RedisError as exc: + raise APIError(503, "unavailable", "Session store unavailable") from exc + raise unauthorized() from None + with db() as conn: + row = conn.execute("SELECT * FROM users WHERE id = %s", (ident,)).fetchone() + if row is None: + raise unauthorized() + return public_user(row) + + +def login(req: Request): + auth = req.headers.get("Authorization", "") + if not auth.startswith("Basic "): + raise unauthorized() + try: + decoded = base64.b64decode(auth[6:].encode("ascii"), validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + if not PASSWORD_RE.fullmatch(username): + raise ValueError() + except (ValueError, UnicodeError, binascii.Error): + raise unauthorized() from None + with db() as conn: + row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone() + if not row: + raise unauthorized() + try: + hasher.verify(row["password_hash"], password) + except (VerifyMismatchError, VerificationError): + raise unauthorized() from None + token = secrets.token_urlsafe(32) + try: + cache().set(session_key(token), str(row["id"]), ex=SESSION_TTL) + except RedisError as exc: + raise APIError(503, "unavailable", "Session store unavailable") from exc + return {"token": token, "token_type": "Bearer", "expires_at": iso(datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)), "user": public_user(row)} + + +def logout(req: Request): + current_user(req) + try: + cache().delete(session_key(bearer_token(req))) + except RedisError as exc: + raise APIError(503, "unavailable", "Session store unavailable") from exc diff --git a/app/core.py b/app/core.py new file mode 100644 index 0000000..7a85e52 --- /dev/null +++ b/app/core.py @@ -0,0 +1,79 @@ +"""Configuration, HTTP errors and shared representation helpers.""" +import base64 +import json +import os +import uuid +from datetime import datetime, timezone + +from fastapi import Request + +INSTANCE_ID = os.getenv("API_INSTANCE_ID", uuid.uuid4().hex[:12]) +SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +IDEMPOTENCY_TTL = max(86400, int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))) + +class APIError(Exception): + def __init__(self, status: int, code: str, message: str): + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +def iso(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def invalid(message: str = "Invalid request") -> APIError: + return APIError(400, "invalid_request", message) + + +def parse_uuid(value: str) -> str: + try: + return str(uuid.UUID(value)) + except (ValueError, TypeError, AttributeError): + raise invalid("Invalid UUID") from None + + +def parse_limit(raw: str | None) -> int: + if raw is None: + return 50 + try: + if not raw.isascii() or not raw.isdecimal(): + raise ValueError() + value = int(raw) + except (ValueError, AttributeError): + raise invalid("Invalid limit") from None + if not 1 <= value <= 100: + raise invalid("Invalid limit") + return value + + +def make_cursor(kind: str, scope: str, created: datetime, ident: str) -> str: + b = json.dumps({"kind": kind, "scope": scope, "created_at": iso(created), "id": ident}, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(b).decode().rstrip("=") + + +def read_cursor(raw: str | None, kind: str, scope: str): + if raw is None: + return None + try: + if not raw or len(raw) > 2048 or any(ch not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" for ch in raw): + raise ValueError() + data = json.loads(base64.b64decode(raw + "=" * (-len(raw) % 4), altchars=b"-_", validate=True)) + if not isinstance(data, dict) or data.get("kind") != kind or data.get("scope") != scope: + raise ValueError() + created = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00")) + if created.tzinfo is None: + raise ValueError() + ident = str(uuid.UUID(data["id"])) + return created, ident + except (ValueError, TypeError, KeyError, UnicodeError, json.JSONDecodeError): + raise invalid("Invalid cursor") from None + + +def request_id(req: Request) -> str: + return getattr(req.state, "request_id", uuid.uuid4().hex) + + +def error_body(req: Request, code: str, message: str): + return {"error": {"code": code, "message": message, "request_id": request_id(req)}} diff --git a/app/image_worker.py b/app/image_worker.py new file mode 100644 index 0000000..9e3f1ee --- /dev/null +++ b/app/image_worker.py @@ -0,0 +1,124 @@ +"""Sandboxed-per-task image work. Process is killed after JOB_TIMEOUT_SECONDS. + +Outputs only JSON variant metadata. Original bytes are copied from durable staging, +never transformed. Pillow reopens/exif-transposes pixels for derivatives, stripping +metadata on re-encode. No API tokens or arbitrary user paths enter this command. +""" +import hashlib +import io +import json +import os +import sys + +from PIL import Image, ImageOps, UnidentifiedImageError +from botocore.exceptions import ClientError, BotoCoreError + +from .attachments import BUCKET, MAX_BYTES, MAX_PIXELS, s3_client +from . import telemetry as ot +from opentelemetry.trace import SpanKind + +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +def rgb_image(src): + orient = ImageOps.exif_transpose(src) + orient.load() + if orient.width * orient.height > MAX_PIXELS: + raise ValueError("too many decoded pixels") + # Compose transparency on white, strip EXIF/ICC and do not enlarge. + rgba = orient.convert("RGBA") + white = Image.new("RGBA", rgba.size, (255, 255, 255, 255)) + white.alpha_composite(rgba) + return white.convert("RGB") + + +def transform(raw): + with Image.open(io.BytesIO(raw)) as image: + if image.format not in ("PNG", "JPEG"): + raise ValueError("invalid encoding") + rgb = rgb_image(image) + t = rgb.copy() + t.thumbnail((256, 256), resample=Image.Resampling.LANCZOS) + # Center crop fit scales up small images when needed. + crop = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) + optim = rgb.copy() + optim.thumbnail((1280, 1280), resample=Image.Resampling.LANCZOS) + return [ + ("thumbnail", t, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}), + ("crop", crop, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}), + ("optimized", optim, "WEBP", "image/webp", {"quality": 78, "method": 4}), + ] + + +def run(attachment_id, pipeline_version): + import psycopg + from psycopg.rows import dict_row + + with ot.span("db.load_staging_metadata", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}): + with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5) as conn: + row = conn.execute(""" + SELECT * FROM attachments WHERE id=%s AND pipeline_version=%s + """, (attachment_id, pipeline_version)).fetchone() + if not row or not row["staging_key"]: + raise ValueError("staging metadata not found") + + client = s3_client() + body = client.get_object(Bucket=BUCKET, Key=row["staging_key"])["Body"] + try: + raw = body.read(MAX_BYTES + 1) + finally: + body.close() + if not raw or len(raw) > MAX_BYTES or len(raw) != row["size_bytes"]: + raise ValueError("staging size mismatch") + if hashlib.sha256(raw).hexdigest() != row["sha256"]: + raise ValueError("staging checksum mismatch") + + # Recompute every result if a prior run was interrupted. Keys are deterministic + # for (attachment_id,pipeline_version,variant), so duplicates don't multiply objects. + output = [] + for name, image, fmt, mime, options in transform(raw): + buf = io.BytesIO() + image.save(buf, format=fmt, **options) + encoded = buf.getvalue() + key = f"variants/{attachment_id}/v{pipeline_version}/{name}" + client.put_object(Bucket=BUCKET, Key=key, Body=encoded, ContentType=mime) + output.append({"name": name, "content_type": mime, + "size_bytes": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(), + "width": image.width, "height": image.height, "object_key": key}) + + # Only after derivatives exist, publish unchanged primary and recovery copy. + client.copy_object(Bucket=BUCKET, Key=row["object_key"], + CopySource={"Bucket": BUCKET, "Key": row["staging_key"]}, + MetadataDirective="REPLACE", ContentType=row["content_type"]) + client.copy_object(Bucket=BUCKET, Key=row["backup_key"], + CopySource={"Bucket": BUCKET, "Key": row["object_key"]}, + MetadataDirective="REPLACE", ContentType=row["content_type"]) + return output + + +def main(): + import uuid + if len(sys.argv) != 3: + raise SystemExit(2) + try: + ident = str(uuid.UUID(sys.argv[1])) + version = int(sys.argv[2]) + if version < 1: + raise ValueError("invalid version") + ot.init("worker", child=True) + with ot.span("image.transform", context=ot.extract(json.loads(os.getenv("LAB6_PARENT_CONTEXT", "{}")))): + out = run(ident, version) + ot.event("image.transformed") + print(json.dumps({"variants": out}, ensure_ascii=False)) + except (ValueError, UnidentifiedImageError, Image.DecompressionBombError, Image.DecompressionBombWarning) as exc: + print("Permanent image processing failure", file=sys.stderr) + raise SystemExit(20) from None + except (BotoCoreError, ClientError, OSError): + print("Transient object storage failure", file=sys.stderr) + raise SystemExit(30) from None + +if __name__ == "__main__": + try: + main() + finally: + ot.flush() diff --git a/app/jobs.py b/app/jobs.py new file mode 100644 index 0000000..504800d --- /dev/null +++ b/app/jobs.py @@ -0,0 +1,470 @@ +"""Durable RabbitMQ delivery + PostgreSQL transactional outbox and worker leases. + +There is no exactly-once transport guarantee. We guarantee eventual recovery of +committed intent and idempotent DB results under at-least-once redelivery. +Commands: `python -m app.jobs dispatch`, `worker`, `replay --id UUID`, `status`. +""" +import argparse +from contextlib import contextmanager +import json +import logging +import os +import random +import subprocess +import sys +import time +import threading +from . import telemetry as ot +from opentelemetry.trace import SpanKind, Status, StatusCode +import uuid +from datetime import datetime, timezone, timedelta + +import pika +import psycopg +from pika.exceptions import AMQPError +from psycopg.rows import dict_row + +log = logging.getLogger("lab5.jobs") +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +QUEUE = "attachment-processing.v1" +MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) +JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) +LEASE_SECONDS = JOB_TIMEOUT + 45 + + +@contextmanager +def connect_db(): + # No credentials, SQL text, or parameters in spans. + with ot.dependency_span('db.job.transaction', kind=SpanKind.CLIENT, attrs={'db.system.name':'postgresql'}): + with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5, options='-c statement_timeout=5000') as conn: + yield conn + + +def job_payload(job_id, attachment_id, version, attempt): + return { + "schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1", + "attachment_id": str(attachment_id), "pipeline_version": version, + "attempt": attempt, "created_at": datetime.now(timezone.utc).isoformat(), + **ot.inject(), + } + + +@contextmanager +def connection(): + params = pika.ConnectionParameters( + host=os.getenv("BROKER_HOST", "broker"), port=5672, + credentials=pika.PlainCredentials(os.environ["BROKER_USER"], os.environ["BROKER_PASSWORD"]), + heartbeat=max(120, JOB_TIMEOUT + 60), blocked_connection_timeout=120, + socket_timeout=5, connection_attempts=3, retry_delay=1) + mq = pika.BlockingConnection(params) + try: + yield mq + finally: + try: + if mq.is_open: + mq.close() + except AMQPError: + pass + + +def setup_channel(conn): + ch = conn.channel() + ch.queue_declare(queue=QUEUE, durable=True) + return ch + + +def queue_outbox(db, job_id, attachment_id, version, attempt, available_at=None): + payload = job_payload(job_id, attachment_id, version, attempt) + if not payload.get('traceparent'): + # Lease rescue/replay has no active consumer context. Preserve the first + # acceptance context rather than inventing a new disconnected trace. + previous = db.execute("SELECT payload FROM job_outbox WHERE job_id=%s ORDER BY created_at,id LIMIT 1", (job_id,)).fetchone() + if previous: + payload.update({k: previous['payload'][k] for k in ('traceparent','tracestate') if k in previous['payload']}) + db.execute(""" + INSERT INTO job_outbox(id,job_id,payload,available_at) + VALUES (%s,%s,%s::jsonb,COALESCE(%s,now())) + """, (str(uuid.uuid4()), job_id, json.dumps(payload), available_at)) + + +def cleanup_abandoned_staging(): + """Best effort cleanup of retained stages; never delete current processing work.""" + from .attachments import BUCKET, s3_client + client = s3_client() + retention = max(3600, int(os.getenv("STAGING_RETENTION_SECONDS", "86400"))) + with connect_db() as db: + rows = db.execute(""" + SELECT id,staging_key FROM attachments + WHERE staging_key IS NOT NULL AND + ((status='ready' AND updated_at < now()-interval '300 seconds') OR + (status='failed' AND updated_at < now()-(%s*interval '1 second'))) + LIMIT 50 + """, (retention,)).fetchall() + for row in rows: + with connect_db() as db: + with db.transaction(): + # Row-level lock serializes against operator replay. + current = db.execute("SELECT status,staging_key,updated_at FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)).fetchone() + if not current or current["staging_key"] != row["staging_key"]: + continue + age = (datetime.now(timezone.utc) - current["updated_at"]).total_seconds() + if not (current["status"] == "ready" and age >= 300 or current["status"] == "failed" and age >= retention): + continue + client.delete_object(Bucket=BUCKET, Key=row["staging_key"]) + db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s", (row["id"], row["staging_key"])) + # Also prune staging uploads whose DB COMMIT failed, after a long retention. + from botocore.exceptions import ClientError + for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET, Prefix="staging/"): + for obj in page.get("Contents", []): + if (datetime.now(timezone.utc) - obj["LastModified"]).total_seconds() < retention: + continue + key = obj["Key"] + with connect_db() as db: + with db.transaction(): + referenced = db.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone() + if not referenced: + client.delete_object(Bucket=BUCKET, Key=key) + +def rescue_expired_jobs(conn): + """Requeue leased jobs whose worker died (durably, no manual task recreation).""" + with conn.transaction(): + rows = conn.execute(""" + SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt + FROM processing_jobs j + WHERE j.status='processing' AND j.leased_until < now() + FOR UPDATE SKIP LOCKED LIMIT 20 + """).fetchall() + for row in rows: + if row["attempt"] >= MAX_ATTEMPTS: + conn.execute("""UPDATE processing_jobs + SET status='failed',error_code='retry_exhausted',lease_token=NULL,leased_until=NULL,updated_at=now() + WHERE id=%s""", (row["id"],)) + conn.execute("""UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now() + WHERE id=%s""", (row["attachment_id"],)) + else: + attempt = row["attempt"] + 1 + conn.execute("""UPDATE processing_jobs SET status='queued',attempt=%s, + lease_token=NULL,leased_until=NULL,next_run_at=now(),updated_at=now() + WHERE id=%s""", (attempt, row["id"])) + conn.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (row["attachment_id"],)) + queue_outbox(conn, row["id"], row["attachment_id"], row["pipeline_version"], attempt) + log.warning("reclaimed timed-out job %s attempt %s", row["id"], attempt) + + +def sample_backlog(): + # Query durable *queued* jobs, including delayed retries and unpublished outbox. + # No broker membership scan; oldest age is wall clock since initial acceptance. + while True: + try: + with psycopg.connect(os.environ['DATABASE_URL'], row_factory=dict_row, connect_timeout=3, options='-c statement_timeout=2000') as db: + row = db.execute("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS age FROM processing_jobs WHERE status='queued'").fetchone() + ot.backlog_depth, ot.backlog_age = row['depth'], float(row['age']) + except psycopg.Error: + ot.backlog_depth = ot.backlog_age = None + time.sleep(3) + + +def dispatch_loop(): + """Publish outbox rows with publisher confirm, mark only AFTER confirm.""" + threading.Thread(target=sample_backlog, daemon=True).start() + while True: + try: + with connection() as mq: + ch = setup_channel(mq) + ch.confirm_delivery() + last_clean = time.monotonic() + last_queue_sample = 0 + while True: + # Periodic lease recovery works even if original message was consumed. + with connect_db() as conn: + rescue_expired_jobs(conn) + with conn.transaction(): + rows = conn.execute(""" + SELECT id,payload FROM job_outbox + WHERE published_at IS NULL AND available_at<=now() + ORDER BY available_at,created_at + FOR UPDATE SKIP LOCKED LIMIT 20 + """).fetchall() + for row in rows: + if os.getenv("LAB5_FAULT_BEFORE_PUBLISH") == "once": + # Persist one-time fault before hard exit; outbox still + # unmarked and dispatcher will publish on restart. + with connect_db() as faultdb: + inserted = faultdb.execute(""" + INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'before_publish') + ON CONFLICT DO NOTHING RETURNING job_id + """, (row["payload"]["job_id"],)).fetchone() + if inserted: + os._exit(78) + payload = row['payload'] + with ot.span('rabbitmq.publish', kind=SpanKind.PRODUCER, context=ot.extract(payload), + attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE,'job.attempt':payload['attempt']}) as producer: + # Propagate actual producer span, while DB retains the original intent. + raw = json.dumps({**payload, **ot.inject()}, separators=(",", ":")) + ch.basic_publish(exchange="", routing_key=QUEUE, body=raw.encode("utf-8"), + mandatory=True, properties=pika.BasicProperties(content_type="application/json", delivery_mode=2)) + conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=%s", (row["id"],)) + ot.event('job.published', job_id=payload['job_id'], attempt=payload['attempt']) + if time.monotonic() - last_queue_sample > 3: + q = ch.queue_declare(queue=QUEUE, passive=True).method + ot.broker_ready, ot.broker_consumers = q.message_count, q.consumer_count + last_queue_sample = time.monotonic() + if time.monotonic() - last_clean > 60: + try: + cleanup_abandoned_staging() + except Exception: + log.warning("staging janitor failed; retry later") + last_clean = time.monotonic() + if not rows: + mq.sleep(1) + except (AMQPError, OSError, psycopg.Error) as exc: + ot.broker_ready = ot.broker_consumers = None + log.warning("dispatcher reconnect after %s", type(exc).__name__) + time.sleep(2) + + +def claim_job(db, message): + job_id = message["job_id"] + with db.transaction(): + job = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job_id,)).fetchone() + if job is None or job["status"] in ("ready", "failed") or job["attempt"] != message["attempt"] or job["pipeline_version"] != message["pipeline_version"]: + return "ignore", None + if job["status"] == "processing": + return "busy", None + if job["next_run_at"] > datetime.now(timezone.utc): + return "busy", None + lease = str(uuid.uuid4()) + db.execute("""UPDATE processing_jobs SET status='processing',lease_token=%s, + leased_until=now()+(%s*interval '1 second'),updated_at=now() + WHERE id=%s""", (lease, LEASE_SECONDS, job_id)) + db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (job["attachment_id"],)) + return "claimed", {"job": job, "lease": lease} + + +def publish_result(db, *, job, lease, variants): + """Final result is committed atomically; stale leases/versions cannot publish.""" + with db.transaction(): + current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone() + if current["status"] != "processing" or str(current["lease_token"]) != lease or current["pipeline_version"] != job["pipeline_version"]: + log.warning("stale worker result ignored %s", job["id"]) + return False + for v in variants: + db.execute(""" + INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) + ON CONFLICT(attachment_id,pipeline_version,name) DO UPDATE SET + object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type, + size_bytes=EXCLUDED.size_bytes,sha256=EXCLUDED.sha256, + width=EXCLUDED.width,height=EXCLUDED.height + """, (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], + v["content_type"], v["size_bytes"], v["sha256"], v["width"], v["height"])) + db.execute("""UPDATE attachments SET status='ready',error_code=NULL,updated_at=now() + WHERE id=%s AND pipeline_version=%s""", (job["attachment_id"], job["pipeline_version"])) + db.execute("""UPDATE processing_jobs SET status='ready',lease_token=NULL,leased_until=NULL, + error_code=NULL,updated_at=now() WHERE id=%s""", (job["id"],)) + return True + + +def fail_job(db, job, lease, *, permanent=False, code="processing_failed"): + with db.transaction(): + current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone() + if current["status"] != "processing" or str(current["lease_token"]) != lease: + return "stale" + if permanent or job["attempt"] >= MAX_ATTEMPTS: + code = code if permanent else "retry_exhausted" + db.execute("""UPDATE processing_jobs SET status='failed',error_code=%s,lease_token=NULL, + leased_until=NULL,updated_at=now() WHERE id=%s""", (code, job["id"])) + db.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s", (code, job["attachment_id"])) + return "failed" + else: + attempt = job["attempt"] + 1 + delay = min(60.0, 2.0 * (2 ** (attempt - 2))) + random.uniform(0, 1.0) + run_at = datetime.now(timezone.utc) + timedelta(seconds=delay) + db.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,next_run_at=%s, + lease_token=NULL,leased_until=NULL,updated_at=now() WHERE id=%s""", (attempt, run_at, job["id"])) + db.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (job["attachment_id"],)) + queue_outbox(db, job["id"], job["attachment_id"], job["pipeline_version"], attempt, run_at) + log.warning("job %s scheduled retry %s after %.1fs", job["id"], attempt, delay) + return "retry" + + +def cleanup_staging(job): + from .attachments import BUCKET, s3_client + from botocore.exceptions import BotoCoreError, ClientError + try: + with connect_db() as conn: + row = conn.execute("SELECT staging_key,status FROM attachments WHERE id=%s", (job["attachment_id"],)).fetchone() + if row and row["status"] == "ready" and row["staging_key"]: + s3_client().delete_object(Bucket=BUCKET, Key=row["staging_key"]) + with connect_db() as conn: + conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND status='ready' AND staging_key=%s", + (job["attachment_id"], row["staging_key"])) + except (BotoCoreError, ClientError, psycopg.Error, OSError): + log.warning("staging cleanup deferred for %s", job["id"]) + + +def process_message(raw): + try: + carrier = json.loads(raw) + if not isinstance(carrier, dict): carrier = {} + except (ValueError, TypeError): + carrier = {} + with ot.span('rabbitmq.consume', kind=SpanKind.CONSUMER, context=ot.extract(carrier), + attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE}) as consumer: + return process_message_inner(raw, consumer) + + +def process_message_inner(raw, consumer): + try: + msg = json.loads(raw) + assert msg["schema_version"] == 1 and msg["type"] == "attachment.process.v1" + uuid.UUID(msg["job_id"]) + uuid.UUID(msg["attachment_id"]) + assert type(msg["attempt"]) is int and msg["attempt"] >= 1 + except (ValueError, KeyError, AssertionError, TypeError): + log.error("invalid internal broker message discarded") + return "ack" # no secrets printed, does not touch DB + with connect_db() as db: + status, state = claim_job(db, msg) + if status == "ignore": + return "ack" + if status == "busy": + return "requeue" + job, lease = state["job"], state["lease"] + consumer.set_attribute('job.attempt',job['attempt']) + consumer.set_attribute('job.id',str(job['id'])) + ot.event('job.claimed', job_id=str(job['id']), attempt=job['attempt']) + start = time.monotonic() + ot.worker_busy = 1 + def done(outcome): + attrs = {'job_type':'attachment.process.v1','outcome':outcome} + ot.count('lab_jobs',attrs) + ot.observe('lab_job_duration',time.monotonic()-start,attrs) + if outcome == 'retry': + ot.count('lab_job_retries',{'job_type':'attachment.process.v1'}) + if outcome in ('retry','failed','requeue'): + consumer.set_status(Status(StatusCode.ERROR)) + consumer.set_attribute('job.outcome',outcome) + ot.event('job.'+outcome, level=logging.ERROR if outcome in ('failed','retry','requeue') else logging.INFO, job_id=str(job['id']), attempt=job['attempt']) + try: + if os.getenv('LAB6_FAULT_RETRY_ONCE') == 'once': + with connect_db() as db: + inserted = db.execute("INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'lab6_retry') ON CONFLICT DO NOTHING RETURNING job_id", (job['id'],)).fetchone() + if inserted: + raise RuntimeError('injected transient retry') + cmd = [sys.executable, "-m", "app.image_worker", str(job["attachment_id"]), str(job["pipeline_version"])] + with ot.span('image.subprocess'): + env = {**os.environ, 'LAB6_PARENT_CONTEXT': json.dumps(ot.inject())} + proc = subprocess.run(cmd, capture_output=True, timeout=JOB_TIMEOUT, check=False, env=env) + if proc.returncode != 0: + with connect_db() as db: + outcome = fail_job(db, job, lease, permanent=(proc.returncode == 20), + code="invalid_image" if proc.returncode == 20 else "processing_failed") + done(outcome) + return "ack" # next attempt is already safely in DB outbox + result = json.loads(proc.stdout) + variants = result["variants"] + if sorted(v["name"] for v in variants) != ["crop", "optimized", "thumbnail"]: + raise ValueError("incomplete variants") + if os.getenv("LAB5_FAULT_AFTER_S3") == "once": + # One-time operator-only injected hard death, after all S3 writes and + # before DB result commit/ack; SQL PK prevents endless failure loop. + with connect_db() as db: + with db.transaction(): + inserted = db.execute(""" + INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'after_s3') + ON CONFLICT DO NOTHING RETURNING job_id + """, (job["id"],)).fetchone() + if inserted: + os._exit(77) + with ot.span("db.publish_result", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}): + with connect_db() as db: + ok = publish_result(db, job=job, lease=lease, variants=variants) + if ok: + ot.observe('lab_ready_duration',(datetime.now(timezone.utc)-job['created_at']).total_seconds(), {'job_type':'attachment.process.v1'}) + cleanup_staging(job) + done('ready' if ok else 'stale') + return "ack" + except subprocess.TimeoutExpired: + with connect_db() as db: + outcome = fail_job(db, job, lease, code="processing_timeout") + done(outcome) + return "ack" + except Exception: + log.exception("job processing failure (internal details not sent to API)") + # Leave lease for dispatcher recovery if even DB unavailable. + try: + with connect_db() as db: + outcome = fail_job(db, job, lease) + done(outcome) + return "ack" + except psycopg.Error: + done("requeue") + return "requeue" + finally: + ot.worker_busy = 0 + + +def worker_loop(): + while True: + try: + with connection() as mq: + ch = setup_channel(mq) + ch.basic_qos(prefetch_count=max(1, int(os.getenv("WORKER_PREFETCH", "1")))) + + def callback(channel, method, props, body): + outcome = process_message(body) + if outcome == "ack": + channel.basic_ack(method.delivery_tag) + else: + channel.basic_nack(method.delivery_tag, requeue=True) + time.sleep(2) # prevent tight redelivery spin on an active lease + + ch.basic_consume(queue=QUEUE, on_message_callback=callback, auto_ack=False) + log.info("worker ready; max attempts=%d timeout=%ds", MAX_ATTEMPTS, JOB_TIMEOUT) + ch.start_consuming() + except (AMQPError, OSError, psycopg.Error) as exc: + log.warning("worker reconnect after %s", type(exc).__name__) + time.sleep(2) + + +def replay_failed(ident): + with connect_db() as db: + with db.transaction(): + row = db.execute("SELECT * FROM processing_jobs WHERE attachment_id=%s FOR UPDATE", (ident,)).fetchone() + if not row or row["status"] != "failed": + raise RuntimeError("Only failed jobs may be explicitly replayed") + attachment = db.execute("SELECT staging_key FROM attachments WHERE id=%s", (ident,)).fetchone() + if not attachment or not attachment["staging_key"]: + raise RuntimeError("Retained staging is required for replay") + db.execute("""UPDATE processing_jobs SET status='queued',attempt=1,next_run_at=now(), + lease_token=NULL,leased_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s""", (row["id"],)) + db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s", (ident,)) + queue_outbox(db, row["id"], ident, row["pipeline_version"], 1) + db.execute("INSERT INTO job_replay_audit(id,job_id) VALUES (%s,%s)", (str(uuid.uuid4()), row["id"])) + print("Replayed failed job", ident) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=["dispatch", "worker", "replay", "status"]) + parser.add_argument("--id", help="UUID for explicit operator replay/status") + args = parser.parse_args() + if args.mode == "dispatch": + ot.init("dispatcher") + dispatch_loop() + elif args.mode == "worker": + ot.init("worker") + worker_loop() + elif args.mode in ("replay", "status"): + if not args.id: + parser.error("--id is required") + ident = str(uuid.UUID(args.id)) + if args.mode == "replay": + replay_failed(ident) + else: + with connect_db() as db: + print(json.dumps(db.execute("SELECT status,attempt,error_code FROM processing_jobs WHERE attachment_id=%s", (ident,)).fetchone(), default=str)) + +if __name__ == "__main__": + main() diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..1fe53e6 --- /dev/null +++ b/app/main.py @@ -0,0 +1,117 @@ +"""FastAPI process: one stateless replica. No auto-migration on startup.""" +import logging +import uuid +import time +from starlette.routing import Match +from opentelemetry.trace import SpanKind, Status, StatusCode +from . import telemetry as ot +from contextlib import asynccontextmanager + +import psycopg +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.concurrency import run_in_threadpool +from redis.exceptions import RedisError + +from .api import router +from .attachments import router as attachment_router +from .auth import challenge, current_user +from .core import APIError, INSTANCE_ID, error_body, invalid, request_id +from .storage import start, stop + +logging.basicConfig(level="INFO") +logger = logging.getLogger("lab5") + +@asynccontextmanager +async def lifespan(app: FastAPI): + ot.init("api") + start() + try: + yield + finally: + stop() + ot.flush() + +app = FastAPI(title="MTUCI Messenger Lab 6", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan) +app.include_router(router) +app.include_router(attachment_router) + +PROTECTED = ("/api/v1/chats", "/api/v1/attachments", "/api/v1/users/me", "/api/v1/auth/sessions/current") +JSON_POST = ("/api/v1/users", "/api/v1/chats") + + +@app.middleware("http") +async def request_context(req: Request, call_next): + # Match the registered template even when early authentication fails. + route = next((r.path for r in app.routes if r.matches(req.scope)[0] == Match.FULL), 'unmatched') + method = req.method if req.method in ('GET','POST','DELETE','PUT','PATCH','HEAD','OPTIONS') else 'OTHER' + attrs = {'method': method, 'route': route} + start_at = time.monotonic() + if route in ("/health/live", "/health/ready"): + response = await handle_request(req, call_next) + attrs["status_class"] = str(response.status_code//100)+"xx" + ot.count("lab_http_requests", attrs) + ot.observe("lab_http_duration", time.monotonic()-start_at, attrs) + return response + with ot.span(method+' '+route, kind=SpanKind.SERVER, attrs={'http.request.method':method,'http.route':route}) as span: + response = await handle_request(req, call_next) + attrs['status_class'] = str(response.status_code//100)+'xx' + span.set_attribute('http.response.status_code', response.status_code) + if response.status_code >= 500: + span.set_status(Status(StatusCode.ERROR)) + ot.count('lab_http_requests', attrs) + ot.observe('lab_http_duration', time.monotonic()-start_at, attrs) + ot.event('http.completed', level=logging.ERROR if response.status_code >= 500 else logging.INFO, + request_id=request_id(req), route=route, method=method, status_code=response.status_code) + if ot.trace_id(): + response.headers['X-Trace-Id'] = ot.trace_id() + return response + + +async def handle_request(req: Request, call_next): + req.state.request_id = uuid.uuid4().hex + try: + path = req.url.path + # Authenticate before parsing the request body or resource-specific fields. + if any(path == p or path.startswith(p + "/") for p in PROTECTED): + req.state.actor = await run_in_threadpool(current_user, req) + if req.method == "POST" and (path in JSON_POST or (path.startswith("/api/v1/chats/") and path.endswith("/messages"))): + if req.headers.get("content-type", "").split(";", 1)[0].strip().lower() != "application/json": + raise APIError(415, "unsupported_media_type", "Content-Type must be application/json") + if req.method == "POST" and path == "/api/v1/auth/sessions" and await req.body(): + raise invalid("Login body is not allowed") + response = await call_next(req) + except APIError as exc: + response = api_error(req, exc) + except (RedisError, psycopg.OperationalError, psycopg.InterfaceError): + response = api_error(req, APIError(503, "unavailable", "Dependency unavailable")) + except Exception: + logger.exception("unexpected request failure id=%s", request_id(req)) + response = api_error(req, APIError(500, "internal_error", "Internal server error")) + response.headers["X-Request-Id"] = request_id(req) + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +def api_error(req: Request, exc: APIError) -> JSONResponse: + headers = {"WWW-Authenticate": challenge(req)} if exc.status == 401 else {} + return JSONResponse(status_code=exc.status, content=error_body(req, exc.code, exc.message), headers=headers) + + +@app.exception_handler(APIError) +async def api_error_handler(req: Request, exc: APIError): + return api_error(req, exc) + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(req: Request, exc: RequestValidationError): + return api_error(req, invalid()) + + +@app.exception_handler(Exception) +async def unexpected_error_handler(req: Request, exc: Exception): + logger.exception("handler failure id=%s", request_id(req)) + if isinstance(exc, (psycopg.OperationalError, psycopg.InterfaceError, RedisError)): + return api_error(req, APIError(503, "unavailable", "Dependency unavailable")) + return api_error(req, APIError(500, "internal_error", "Internal server error")) diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 0000000..5f624e3 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,23 @@ +"""Run once before API instances; transactional and advisory-lock coordinated migrations.""" +import os +from pathlib import Path +import psycopg + +MIGRATIONS = Path(__file__).resolve().parents[1] / "db" / "migrations" + +def main(): + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=False) as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(62132103)") + conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())") + for path in sorted(MIGRATIONS.glob("[0-9]*.sql")): + applied = conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone() + if applied: + continue + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (path.name,)) + print("Applied", path.name, flush=True) + print("Migrations up to date", flush=True) + +if __name__ == "__main__": + main() diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..d0018c5 --- /dev/null +++ b/app/models.py @@ -0,0 +1,77 @@ +"""HTTP request models; all strings use Unicode code point limits.""" +import re +import uuid +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + +class CreateUser(StrictModel): + username: str + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("username") + @classmethod + def username_valid(cls, value: str) -> str: + if not USERNAME_RE.fullmatch(value): + raise ValueError("Invalid username") + return value + + @field_validator("display_name", "password") + @classmethod + def non_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("Blank value") + value.encode("utf-8", errors="strict") + return value + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def non_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("Blank title") + value.encode("utf-8", errors="strict") + return value + + @field_validator("member_ids") + @classmethod + def uuid_members(cls, value: list[str]) -> list[str]: + for member in value: + if not isinstance(member, str) or str(uuid.UUID(member)) != member: + raise ValueError("Invalid member UUID") + if len(value) != len(set(value)): + raise ValueError("Duplicate member") + return value + +class CreateMessage(StrictModel): + text: str = Field(min_length=0, max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("text") + @classmethod + def non_blank(cls, value: str) -> str: + value.encode("utf-8", errors="strict") + return value + + @field_validator("attachment_ids") + @classmethod + def valid_attachments(cls, value: list[str]) -> list[str]: + if len(value) != len(set(value)): + raise ValueError("Duplicate attachment IDs") + for item in value: + if not isinstance(item, str) or str(uuid.UUID(item)) != item: + raise ValueError("Invalid attachment UUID") + return value + + @model_validator(mode="after") + def nonempty_message(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Empty message") + return self diff --git a/app/pipeline_selftest.py b/app/pipeline_selftest.py new file mode 100644 index 0000000..af559cb --- /dev/null +++ b/app/pipeline_selftest.py @@ -0,0 +1,25 @@ +"""Offline image conversion tests inside the application image; no S3 or DB.""" +import io +from PIL import Image +from .image_worker import transform + + +def main(): + source = Image.new('RGB',(320,200), (255,0,0)) + buf=io.BytesIO();source.save(buf,'PNG') + result=transform(buf.getvalue()) + items={name:(img,fmt,mime,opts) for name,img,fmt,mime,opts in result} + assert set(items)=={'thumbnail','crop','optimized'} + assert items['thumbnail'][0].size==(256,160) + assert items['crop'][0].size==(128,128) + assert items['optimized'][0].size==(320,200) + for name,(im,fmt,mime,opts) in items.items(): + data=io.BytesIO();im.save(data,format=fmt,**opts) + with Image.open(io.BytesIO(data.getvalue())) as decoded: + assert decoded.size==im.size + assert decoded.format==fmt + assert 'exif' not in decoded.info + print('PASS: PNG 320x200 -> JPEG thumb 256x160, crop 128x128, optimized WebP, metadata stripped') + +if __name__=='__main__': + main() diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 0000000..0f31682 --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,132 @@ +"""Lab4 grade-5 track 2: repeatable DB/S3 reconciliation. + +Dry-run is default. Apply never drops metadata when BOTH object copies are gone: +without a trustworthy copy or external backup, deletion would destroy evidence. +""" +import argparse +import json +import sys +import time +from datetime import datetime, timezone + +import psycopg +from psycopg.rows import dict_row +from botocore.exceptions import BotoCoreError, ClientError + +from .attachments import BUCKET, s3_client + + +def inventory(client): + result = {} + for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET): + for obj in page.get("Contents", []): + # Only reconciliation-owned prefixes; never mutate arbitrary S3 keys. + key = obj["Key"] + if key.startswith(("attachments/", "recovery/")): + result[key] = obj["LastModified"] + return result + + +def get_db_rows(conn): + return conn.execute("SELECT id, object_key, backup_key, status FROM attachments").fetchall() + + +def current_reference(conn, key): + return conn.execute( + "SELECT 1 FROM attachments WHERE object_key=%s OR backup_key=%s", (key, key) + ).fetchone() is not None + + +def reconcile(*, apply=False, grace_seconds=300, now=None, client=None, connect=None): + if grace_seconds < 300 and apply: + raise ValueError("Apply requires >=300 s grace period to protect in-flight uploads") + started = time.monotonic() + now = now or datetime.now(timezone.utc) + client = client or s3_client() + connect = connect or (lambda: psycopg.connect(__import__('os').environ['DATABASE_URL'], row_factory=dict_row)) + objects = inventory(client) + stats = { + "mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds, + "s3_keys_scanned": len(objects), "db_rows_scanned": 0, + "orphan_keys": [], "orphan_keys_young": [], + "missing_primary_repairable": [], "missing_backup_repairable": [], + "missing_both_unrecoverable": [], "actions": [], "errors": [], + } + with connect() as conn: + rows = get_db_rows(conn) + stats["db_rows_scanned"] = len(rows) + references = {key for row in rows for key in (row["object_key"], row["backup_key"])} + for row in rows: + if row["status"] != "ready": + continue # queued/processing have durable staging, not final variants yet + primary, backup = row["object_key"], row["backup_key"] + have_primary, have_backup = primary in objects, backup in objects + rid = str(row["id"]) + if have_primary and have_backup: + continue + if not have_primary and not have_backup: + stats["missing_both_unrecoverable"].append(rid) + continue + dest, source, typ = (primary, backup, "missing_primary_repairable") if not have_primary else (backup, primary, "missing_backup_repairable") + stats[typ].append(rid) + if apply: + # Defensive recheck. Object may have reappeared after inventory. + try: + client.head_object(Bucket=BUCKET, Key=dest) + stats["actions"].append({"action": "already-restored", "key": dest}) + continue + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code not in ("404", "NotFound", "NoSuchKey"): + raise + try: + client.copy_object(Bucket=BUCKET, Key=dest, + CopySource={"Bucket": BUCKET, "Key": source}) + stats["actions"].append({"action": "restore", "key": dest, "from": source}) + except (BotoCoreError, ClientError) as exc: + stats["errors"].append({"action": "restore", "key": dest, "error_type": type(exc).__name__}) + for key, modified in objects.items(): + if key in references: + continue + age = (now - modified).total_seconds() + if age < grace_seconds: + stats["orphan_keys_young"].append(key) + continue + stats["orphan_keys"].append(key) + if apply: + # Query again immediately before destructive action; never delete + # an object now referenced by a committed metadata row. + if current_reference(conn, key): + stats["actions"].append({"action": "skip-now-referenced", "key": key}) + continue + try: + client.delete_object(Bucket=BUCKET, Key=key) + stats["actions"].append({"action": "delete-old-orphan", "key": key}) + except (BotoCoreError, ClientError) as exc: + stats["errors"].append({"action": "delete-old-orphan", "key": key, "error_type": type(exc).__name__}) + stats["scan_elapsed_seconds"] = round(time.monotonic() - started, 3) + return stats + + +def main(): + parser = argparse.ArgumentParser(description="Reconcile S3 attachments vs PostgreSQL (dry-run default)") + parser.add_argument("--apply", action="store_true", help="Make safe changes; orphans older than 300 seconds only") + parser.add_argument("--grace-seconds", type=int, default=300) + parser.add_argument("--output", help="Write JSON summary (no credentials)") + args = parser.parse_args() + try: + result = reconcile(apply=args.apply, grace_seconds=args.grace_seconds) + except Exception as exc: + # No exception string: botocore errors may contain signed credentials/URLs. + print(json.dumps({"status":"failed", "error_type":type(exc).__name__}), file=sys.stderr) + raise SystemExit(2) from None + rendered = json.dumps(result, ensure_ascii=False, indent=2) + print(rendered) + if args.output: + with open(args.output, "w", encoding="utf-8") as fh: + fh.write(rendered + "\n") + if result["errors"] or result["missing_both_unrecoverable"]: + raise SystemExit(1) + +if __name__ == "__main__": + main() diff --git a/app/reconcile_fault.py b/app/reconcile_fault.py new file mode 100644 index 0000000..fce1eea --- /dev/null +++ b/app/reconcile_fault.py @@ -0,0 +1,36 @@ +"""Operator-only fault injection in a LOCAL TEST stack; never exposed as HTTP.""" +import argparse +import json +import os +import uuid +import psycopg +from psycopg.rows import dict_row +from .attachments import BUCKET, s3_client + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("action", choices=["delete-primary", "delete-backup", "create-orphan"]) + p.add_argument("--id", help="Attachment UUID for the delete actions") + args = p.parse_args() + if os.environ.get("LAB4_ALLOW_FAULT_INJECTION") != "yes": + raise SystemExit("Fault injection disabled; use LAB4_ALLOW_FAULT_INJECTION=yes in isolated test") + client = s3_client() + if args.action == "create-orphan": + key = f"attachments/{uuid.uuid4()}/original" + client.put_object(Bucket=BUCKET, Key=key, Body=b"synthetic orphan") + print(json.dumps({"fault":"orphan-created", "object_key":key})) + return + if not args.id: + raise SystemExit("--id is required") + ident = str(uuid.UUID(args.id)) + with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row) as conn: + row = conn.execute("SELECT object_key,backup_key FROM attachments WHERE id=%s", (ident,)).fetchone() + if not row: + raise SystemExit("Attachment does not exist") + key = row["object_key"] if args.action == "delete-primary" else row["backup_key"] + client.delete_object(Bucket=BUCKET, Key=key) + print(json.dumps({"fault":args.action, "object_key":key})) + +if __name__ == "__main__": + main() diff --git a/app/reconcile_selftest.py b/app/reconcile_selftest.py new file mode 100644 index 0000000..e9d2c81 --- /dev/null +++ b/app/reconcile_selftest.py @@ -0,0 +1,61 @@ +"""Fast isolated tests for reconciliation safety; run in application image.""" +import uuid +from datetime import datetime, timezone, timedelta +from .reconcile import reconcile +from .attachments import BUCKET + +NOW = datetime.now(timezone.utc) + +class FakeConn: + def __init__(self, rows): self.rows=rows + def __enter__(self): return self + def __exit__(self,*a): return False + def execute(self, sql, args=None): + self.sql=sql + self.args=args + return self + def fetchall(self): return self.rows + def fetchone(self): + key=self.args[0] + return {'id': 1} if any(key in (row['object_key'],row['backup_key']) for row in self.rows) else None + +class FakeS3: + def __init__(self,objects): self.objects=objects + def get_paginator(self,op): return self + def paginate(self,**kwargs): + yield {'Contents':[{'Key':k,'LastModified':t} for k,t in self.objects.items()]} + def head_object(self,**kw): + if kw['Key'] not in self.objects: + from botocore.exceptions import ClientError + raise ClientError({'Error':{'Code':'404'}},'HeadObject') + def copy_object(self,**kw): + assert kw['CopySource']['Key'] in self.objects + self.objects[kw['Key']]=NOW + def delete_object(self,**kw): + self.objects.pop(kw['Key'],None) + + +def test(): + i=str(uuid.uuid4()) + primary=f'attachments/{i}/original';backup=f'recovery/{i}/original' + young=f'attachments/{uuid.uuid4()}/original' + old=f'attachments/{uuid.uuid4()}/original' + rows=[{'id':i,'object_key':primary,'backup_key':backup,'status':'ready'}] + fake=FakeS3({backup: NOW-timedelta(days=1),young:NOW-timedelta(seconds=10),old:NOW-timedelta(days=1)}) + kwargs={'now':NOW,'client':fake,'connect':lambda:FakeConn(rows)} + first=reconcile(apply=False,**kwargs) + assert first['missing_primary_repairable']==[i] + assert old in first['orphan_keys'] and young in first['orphan_keys_young'] + assert primary not in fake.objects and old in fake.objects, 'Dry run mutated objects' + second=reconcile(apply=True,**kwargs) + assert primary in fake.objects and old not in fake.objects and young in fake.objects + assert any(x['action']=='restore' for x in second['actions']) + third=reconcile(apply=True,**kwargs) + assert not third['actions'], 'Re-run unexpectedly mutated objects' + assert rows[0]['object_key']==primary, 'Metadata destroyed' + try: reconcile(apply=True, grace_seconds=0, **kwargs) + except ValueError: pass + else: raise AssertionError('Unsafe zero-grace apply accepted') + print('PASS: dry-run, orphan grace, recover primary from backup, apply idempotence') + +if __name__ == '__main__': test() diff --git a/app/s3_bootstrap.py b/app/s3_bootstrap.py new file mode 100644 index 0000000..da81f64 --- /dev/null +++ b/app/s3_bootstrap.py @@ -0,0 +1,35 @@ +"""Idempotently create the private S3 bucket with bootstrap-only credentials.""" +import os +import time +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError, BotoCoreError + + +def main(): + bucket = os.environ["S3_BUCKET"] + client = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ADMIN_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_ADMIN_SECRET_KEY"], + region_name="us-east-1", + config=Config(s3={"addressing_style": "path"}, + connect_timeout=2, read_timeout=4, retries={"max_attempts": 1})) + for attempt in range(70): + try: + try: + client.head_bucket(Bucket=bucket) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code not in ("404", "NoSuchBucket", "NotFound"): + raise + client.create_bucket(Bucket=bucket) + # PRIVATE: no public bucket policy, ACL or anonymous identity. + print(f"Private bucket {bucket} ready", flush=True) + return + except (ClientError, BotoCoreError, OSError) as exc: + if attempt == 69: + raise RuntimeError("S3 bootstrap failed; inspect S3 container") from exc + time.sleep(1) + +if __name__ == "__main__": + main() diff --git a/app/s3_policy.py b/app/s3_policy.py new file mode 100644 index 0000000..b51dead --- /dev/null +++ b/app/s3_policy.py @@ -0,0 +1,32 @@ +"""One-off creation of private SeaweedFS S3 identities; secrets stay in Docker volume.""" +import json +import os +from pathlib import Path + + +def main(): + bucket = os.environ["S3_BUCKET"] + identities = [ + {"name": "bucket-bootstrap", "credentials": [{"accessKey": os.environ["S3_ADMIN_ACCESS_KEY"], + "secretKey": os.environ["S3_ADMIN_SECRET_KEY"]}], + "actions": ["Admin", "Read", "Write", "List"]}, + {"name": "messenger-api", "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY"], + "secretKey": os.environ["S3_SECRET_KEY"]}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]}, + ] + if len({entry["credentials"][0]["accessKey"] for entry in identities}) != 2: + raise ValueError("Admin and API credentials must differ") + path = Path("/s3-config/config.json") + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + old_umask = os.umask(0o077) + try: + tmp.write_text(json.dumps({"identities": identities}, separators=(",", ":")), encoding="utf-8") + os.chmod(tmp, 0o644) # readable by the SeaweedFS runtime UID; Docker volume stays private + os.replace(tmp, path) + finally: + os.umask(old_umask) + print("S3 identities configured (no secrets printed)", flush=True) + +if __name__ == "__main__": + main() diff --git a/app/s3_probe.py b/app/s3_probe.py new file mode 100644 index 0000000..ef6d441 --- /dev/null +++ b/app/s3_probe.py @@ -0,0 +1,40 @@ +"""Verify bucket is private at S3 boundary (not merely hidden behind HTTP API).""" +import json +import os +import boto3 +from botocore import UNSIGNED +from botocore.config import Config +from botocore.exceptions import ClientError +from .attachments import BUCKET, s3_client + + +def main(): + client = s3_client() + client.head_bucket(Bucket=BUCKET) # restricted app identity works + unauth = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], + region_name="us-east-1", config=Config(signature_version=UNSIGNED, + s3={"addressing_style":"path"})) + try: + unauth.list_objects_v2(Bucket=BUCKET, MaxKeys=1) + except ClientError as exc: + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status not in (401,403): + raise RuntimeError("Unexpected anonymous S3 response") from None + else: + raise RuntimeError("Private bucket leaked object list to anonymous caller") + listing = client.list_objects_v2(Bucket=BUCKET, MaxKeys=1) + keys = [entry["Key"] for entry in listing.get("Contents", [])] + if keys: + try: + unauth.get_object(Bucket=BUCKET, Key=keys[0]) + except ClientError as exc: + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status not in (401, 403): + raise RuntimeError("Unexpected anonymous object GET response") from None + else: + raise RuntimeError("Private S3 object downloadable anonymously") + print(json.dumps({"private_bucket": True, "anonymous_listing": "denied", + "anonymous_object_get": "denied" if keys else "not_tested_empty_bucket"})) + +if __name__ == "__main__": + main() diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 0000000..b776c8d --- /dev/null +++ b/app/storage.py @@ -0,0 +1,67 @@ +"""Shared external state: psycopg connection pool and Redis session store.""" +import os +from contextlib import contextmanager + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +from redis import Redis +from redis.exceptions import RedisError + +from .core import APIError +from . import telemetry as ot +from opentelemetry.trace import SpanKind + +pool: ConnectionPool | None = None +redis_client: Redis | None = None + + +def start() -> None: + global pool, redis_client + url = os.environ["DATABASE_URL"] + size = int(os.getenv("DB_POOL_MAX", "8")) + pool = ConnectionPool(conninfo=url, min_size=1, max_size=size, timeout=3, kwargs={"row_factory": dict_row, "connect_timeout": 5, "options": "-c statement_timeout=5000"}, open=False) + pool.open(wait=True, timeout=30) + redis_client = Redis.from_url(os.environ.get("REDIS_URL", "redis://redis:6379/0"), socket_connect_timeout=2, socket_timeout=2, decode_responses=True) + redis_client.ping() + + +def stop() -> None: + global pool, redis_client + if pool is not None: + pool.close() + pool = None + if redis_client is not None: + redis_client.close() + redis_client = None + + +@contextmanager +def db(): + if pool is None: + raise APIError(503, "unavailable", "Database unavailable") + try: + with ot.dependency_span("db.transaction", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}): + with pool.connection() as conn: + yield conn + except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled, PoolTimeout) as exc: + raise APIError(503, "unavailable", "Database unavailable") from exc + + +def cache() -> Redis: + if redis_client is None: + raise APIError(503, "unavailable", "Session store unavailable") + return redis_client + + +def ready() -> bool: + try: + with db() as conn: + conn.execute("SELECT 1").fetchone() + cache().ping() + # S3 is needed for the mandatory synchronous upload/download path. + from .attachments import s3_client, BUCKET + s3_client().head_bucket(Bucket=BUCKET) + return True + except Exception: + return False diff --git a/app/telemetry.py b/app/telemetry.py new file mode 100644 index 0000000..df2c06c --- /dev/null +++ b/app/telemetry.py @@ -0,0 +1,204 @@ +"""Explicit, allowlisted OTLP instrumentation; no automatic headers, SQL or bodies. +SDK exporters run asynchronously with bounded queues; telemetry is best effort. +""" +import logging +import os +import socket +import time +from contextlib import contextmanager, nullcontext +from pathlib import Path + +from opentelemetry import metrics, trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + +PROP = TraceContextTextMapPropagator() +tracer = trace.get_tracer('messenger.lab6') +log = logging.getLogger('messenger.telemetry') +log.setLevel(logging.INFO) +log.propagate = False +providers = [] +started = False +m = None +instruments = {} +worker_busy = 0 +backlog_depth = None +backlog_age = None +broker_ready = None +broker_consumers = None +QUEUE = 'attachment-processing.v1' +BUCKETS = (0.005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 15, 30, 60, 120, 300, 600) + + +def init(role, *, child=False): + global started, tracer, m + if started or os.getenv('OTEL_ENABLED', 'no') != 'yes': + return + started = True + resource = Resource.create({'service.name': 'messenger-' + role, + 'service.version': '1.6.0', 'service.instance.id': os.getenv('API_INSTANCE_ID', socket.gethostname()), + 'deployment.environment.name': 'lab'}) + base = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/') + tp = TracerProvider(resource=resource) + tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=base+'/v1/traces', timeout=2), + max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000)) + trace.set_tracer_provider(tp) + tracer = tp.get_tracer('messenger.lab6') + lp = LoggerProvider(resource=resource) + lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=base+'/v1/logs', timeout=2), + max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000)) + log.addHandler(LoggingHandler(logger_provider=lp)) + providers.extend([tp, lp]) + if child: + return + reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=base+'/v1/metrics', timeout=2), + export_interval_millis=3000, export_timeout_millis=2000) + mp = MeterProvider(resource=resource, metric_readers=[reader], views=[View(instrument_name=n, + aggregation=ExplicitBucketHistogramAggregation(boundaries=BUCKETS)) for n in + ('lab_http_duration','lab_job_duration','lab_ready_duration')]) + metrics.set_meter_provider(mp) + providers.append(mp) + m = mp.get_meter('messenger.lab6') + for name in ('lab_http_requests','lab_jobs','lab_job_retries'): + instruments[name] = m.create_counter(name) + for name in ('lab_http_duration','lab_job_duration','lab_ready_duration'): + instruments[name] = m.create_histogram(name, unit='s') + if role == 'api': + for name, key in [('lab_db_pool_used','used'),('lab_db_pool_waiting','requests_waiting'),('lab_db_pool_capacity','pool_max')]: + m.create_observable_gauge(name, callbacks=[lambda _, key=key: pool_stats(key)]) + if role == 'worker': + m.create_observable_gauge('lab_worker_busy', callbacks=[lambda _: [metrics.Observation(worker_busy)]]) + m.create_observable_gauge('lab_worker_capacity', callbacks=[lambda _: [metrics.Observation(1)]]) + if role == 'dispatcher': + m.create_observable_gauge('lab_broker_ready', callbacks=[lambda _: queue_observation(broker_ready)]) + m.create_observable_gauge('lab_broker_consumers', callbacks=[lambda _: queue_observation(broker_consumers)]) + m.create_observable_gauge('lab_queue_depth', callbacks=[lambda _: queue_observation(backlog_depth)]) + m.create_observable_gauge('lab_queue_oldest_age', unit='s', callbacks=[lambda _: queue_observation(backlog_age)]) + m.create_observable_gauge('lab_container_memory', unit='By', callbacks=[lambda _: container_observation('memory')]) + m.create_observable_gauge('lab_container_cpu_cores', callbacks=[lambda _: container_observation('cpu')]) + + +def pool_stats(key): + from . import storage + if storage.pool is None: + return [] + stats = storage.pool.get_stats() + val = stats.get('pool_size',0)-stats.get('pool_available',0) if key=='used' else stats.get(key,0) + return [metrics.Observation(max(0,val))] + + +def queue_observation(value): + return [] if value is None else [metrics.Observation(value, {'queue': QUEUE})] + + +_cpu_previous = None + +def container_observation(kind): + global _cpu_previous + try: + if kind == 'memory': + return [metrics.Observation(int(Path('/sys/fs/cgroup/memory.current').read_text()))] + cpu = int(dict(line.split() for line in Path('/sys/fs/cgroup/cpu.stat').read_text().splitlines())['usage_usec']) / 1e6 + now = time.monotonic() + old = _cpu_previous + _cpu_previous = (now,cpu) + return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))] + except (OSError, KeyError, ValueError): + # cgroup v1 fallback, process tree (not full container accounting). + import psutil + try: + p = psutil.Process() + processes = [p]+p.children(recursive=True) + if kind == 'memory': + value = 0 + for proc in processes: + try: value += proc.memory_info().rss + except psutil.Error: pass + return [metrics.Observation(value)] + # cgroup v1 (Docker Linux) if available, CPU usage in nanoseconds. + for path in ('/sys/fs/cgroup/cpuacct/cpuacct.usage','/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage'): + if Path(path).is_file(): + cpu = int(Path(path).read_text()) / 1e9 + now = time.monotonic() + old = _cpu_previous + _cpu_previous = (now,cpu) + return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))] + except (psutil.Error,OSError,ValueError): + pass + return [] + + +def count(name, attrs=None): + if name in instruments: instruments[name].add(1, attrs or {}) + + +def observe(name, value, attrs=None): + if name in instruments: instruments[name].record(max(0,value), attrs or {}) + + +def event(name, *, level=logging.INFO, **attrs): + # Callers pass fixed event names and technical ID/status only, never payloads. + if started: + log.log(level, name, extra={'event':name, **attrs}) + + +@contextmanager +def span(name, *, kind=SpanKind.INTERNAL, context=None, attrs=None): + with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attrs, + record_exception=False, set_status_on_exception=False) as s: + try: + yield s + except BaseException as exc: + s.set_status(Status(StatusCode.ERROR)) + s.set_attribute('error.type', type(exc).__name__) + event('operation.error', level=logging.ERROR, operation=name, error_type=type(exc).__name__) + raise + + +def inject(): + result = {} + PROP.inject(result) + return result + + +def extract(carrier): + # Ignore baggage and all unrelated fields. + return PROP.extract({k:v for k,v in carrier.items() if k in ('traceparent','tracestate') and isinstance(v,str)}) + + +def dependency_span(name, **kwargs): + return span(name, **kwargs) if trace_id() else nullcontext() + + +def trace_id(): + ctx = trace.get_current_span().get_span_context() + return f'{ctx.trace_id:032x}' if ctx.is_valid else '' + + +def flush(): + for p in providers: + p.force_flush(timeout_millis=2500) + + +class S3Tracing: + """Only operation name is exported; bucket/key/bytes/signed URLs never exported.""" + def __init__(self, client): self.client = client + def __getattr__(self, name): + f = getattr(self.client,name) + if name not in ('upload_fileobj','get_object','put_object','copy_object','delete_object','head_object','head_bucket'): + return f + def call(*a, **kw): + with dependency_span('s3.'+name, kind=SpanKind.CLIENT, attrs={'storage.system':'s3'}): + return f(*a, **kw) + return call diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..3245e2d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,347 @@ +services: + postgres: + image: postgres:16.10-bookworm + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-lab} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: ["postgres_data:/var/lib/postgresql/data"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + cpus: 1.0 + restart: unless-stopped + networks: [storage] + + redis: + image: redis:7.4.5-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", "60", "1"] + volumes: ["redis_data:/data"] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 256m + cpus: 0.5 + restart: unless-stopped + networks: [storage] + + broker: + image: rabbitmq:4.1.8-management-alpine + hostname: lab6-broker + environment: + RABBITMQ_DEFAULT_USER: ${BROKER_USER:-labworker} + RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Set BROKER_PASSWORD in .env} + volumes: ["rabbitmq_data:/var/lib/rabbitmq"] + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 25 + start_period: 15s + mem_limit: 512m + cpus: 0.75 + restart: unless-stopped + networks: [storage] + + s3-policy: + build: . + user: "0:0" + command: ["python", "-m", "app.s3_policy"] + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:?Set S3_ADMIN_ACCESS_KEY} + S3_ADMIN_SECRET_KEY: ${S3_ADMIN_SECRET_KEY:?Set S3_ADMIN_SECRET_KEY} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} + volumes: ["s3_config:/s3-config"] + restart: "no" + networks: [storage] + + s3: + image: chrislusf/seaweedfs:4.53 + command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.port=8333", "-s3.iam=false", "-s3.config=/s3-config/config.json"] + volumes: + - s3_data:/data + - s3_config:/s3-config:ro + - ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro + depends_on: + s3-policy: + condition: service_completed_successfully + expose: ["8333"] + mem_limit: 640m + cpus: 1.0 + restart: unless-stopped + networks: [storage] + + s3-bootstrap: + build: . + command: ["python", "-m", "app.s3_bootstrap"] + environment: + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:?Set S3_ADMIN_ACCESS_KEY} + S3_ADMIN_SECRET_KEY: ${S3_ADMIN_SECRET_KEY:?Set S3_ADMIN_SECRET_KEY} + depends_on: + s3: + condition: service_started + restart: "no" + networks: [storage] + + migrate: + build: . + command: ["python", "-m", "app.migrate"] + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger} + depends_on: + postgres: + condition: service_healthy + restart: "no" + networks: [storage] + + reconcile: + profiles: [maintenance] + build: . + command: ["python", "-m", "app.reconcile"] + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} + depends_on: + postgres: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + restart: "no" + networks: [storage] + + dispatcher: + build: . + command: ["python", "-m", "app.jobs", "dispatch"] + environment: &job_env + OTEL_ENABLED: ${OTEL_ENABLED:-yes} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger} + BROKER_HOST: broker + BROKER_USER: ${BROKER_USER:-labworker} + BROKER_PASSWORD: ${BROKER_PASSWORD:?Set BROKER_PASSWORD in .env} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + depends_on: + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + mem_limit: 192m + cpus: 0.5 + restart: unless-stopped + networks: [storage, telemetry] + + worker: + build: . + command: ["python", "-m", "app.jobs", "worker"] + environment: + <<: *job_env + LAB6_FAULT_RETRY_ONCE: ${LAB6_FAULT_RETRY_ONCE:-off} + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + LAB5_FAULT_AFTER_S3: ${LAB5_FAULT_AFTER_S3:-off} + depends_on: + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + mem_limit: 768m + cpus: 1.0 + stop_grace_period: 75s + restart: unless-stopped + networks: [storage, telemetry] + + api1: + build: . + environment: &api_env + OTEL_ENABLED: ${OTEL_ENABLED:-yes} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + APP_PORT: "8080" + API_INSTANCE_ID: api1 + DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger} + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_POOL_MAX: ${DB_POOL_MAX:-8} + JOB_BACKLOG_LIMIT: ${JOB_BACKLOG_LIMIT:-1000} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} + depends_on: &api_depends + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + expose: ["8080"] + healthcheck: &api_health + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 2s + timeout: 3s + retries: 15 + start_period: 10s + mem_limit: 384m + cpus: 0.75 + stop_grace_period: 12s + restart: unless-stopped + networks: [frontend, storage, telemetry] + + api2: + build: . + environment: + <<: *api_env + API_INSTANCE_ID: api2 + depends_on: *api_depends + expose: ["8080"] + healthcheck: *api_health + mem_limit: 384m + cpus: 0.75 + stop_grace_period: 12s + restart: unless-stopped + networks: [frontend, storage, telemetry] + + lb: + image: haproxy:3.1.7-alpine + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + ports: ["127.0.0.1:8080:8080"] + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + healthcheck: + test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + mem_limit: 128m + cpus: 0.5 + restart: unless-stopped + networks: [frontend] + + telemetry-init: + image: busybox:1.37.0 + user: "0:0" + command: ["sh", "-ec", "chown 10001:10001 /tempo /loki; chown 65534:65534 /prometheus; chown 472:472 /grafana"] + volumes: + - tempo_data:/tempo + - loki_data:/loki + - prometheus_data:/prometheus + - grafana_data:/grafana + network_mode: none + restart: "no" + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.123.0 + command: ["--config=/etc/otelcol/config.yaml"] + volumes: ["./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro"] + environment: + GOMEMLIMIT: 180MiB + mem_limit: 256m + cpus: 0.75 + restart: unless-stopped + networks: [telemetry] + + prometheus: + depends_on: + telemetry-init: + condition: service_completed_successfully + image: prom/prometheus:v3.2.1 + command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=72h", "--storage.tsdb.retention.size=512MB"] + volumes: + - ./infra/otel/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/otel/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - prometheus_data:/prometheus + ports: ["127.0.0.1:9090:9090"] + mem_limit: 384m + restart: unless-stopped + networks: [telemetry] + + loki: + depends_on: + telemetry-init: + condition: service_completed_successfully + image: grafana/loki:3.4.2 + command: ["-config.file=/etc/loki/local-config.yaml"] + volumes: + - ./infra/otel/loki.yaml:/etc/loki/local-config.yaml:ro + - loki_data:/loki + ports: ["127.0.0.1:3100:3100"] + mem_limit: 512m + restart: unless-stopped + networks: [telemetry] + + tempo: + depends_on: + telemetry-init: + condition: service_completed_successfully + image: grafana/tempo:2.7.2 + command: ["-config.file=/etc/tempo.yaml"] + volumes: + - ./infra/otel/tempo.yaml:/etc/tempo.yaml:ro + - tempo_data:/var/tempo + ports: ["127.0.0.1:3200:3200"] + mem_limit: 512m + restart: unless-stopped + networks: [telemetry] + + grafana: + depends_on: + telemetry-init: + condition: service_completed_successfully + image: grafana/grafana:11.6.0 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + volumes: + - grafana_data:/var/lib/grafana + - ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: ["127.0.0.1:3000:3000"] + mem_limit: 384m + restart: unless-stopped + networks: [telemetry] + +networks: + telemetry: + frontend: + storage: + internal: true +volumes: + postgres_data: + redis_data: + s3_data: + s3_config: + rabbitmq_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: diff --git a/db/migrations/001_initial.sql b/db/migrations/001_initial.sql new file mode 100644 index 0000000..203b012 --- /dev/null +++ b/db/migrations/001_initial.sql @@ -0,0 +1,33 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name BYTEA NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_display_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400) +); +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400) +); +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id), + user_id UUID NOT NULL REFERENCES users(id), + position INTEGER NOT NULL, + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); +CREATE INDEX IF NOT EXISTS idx_members_user ON chat_members (user_id, chat_id); +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + sender_id UUID NOT NULL, + text BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id), + CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000) +); +CREATE INDEX IF NOT EXISTS idx_messages_chat_history ON messages (chat_id, created_at, id); +CREATE INDEX IF NOT EXISTS idx_chats_history ON chats (created_at, id); diff --git a/db/migrations/002_idempotency.sql b/db/migrations/002_idempotency.sql new file mode 100644 index 0000000..04a000a --- /dev/null +++ b/db/migrations/002_idempotency.sql @@ -0,0 +1,12 @@ +-- Same transaction creates message and idempotency receipt. Unique key arbitrates concurrent replicas. +CREATE TABLE IF NOT EXISTS message_idempotency ( + actor_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + method VARCHAR(8) NOT NULL DEFAULT 'POST', + idem_key VARCHAR(128) NOT NULL, + body_sha256 CHAR(64) NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (actor_id, chat_id, method, idem_key) +); +CREATE INDEX IF NOT EXISTS idx_idempotency_age ON message_idempotency (created_at); diff --git a/db/migrations/003_attachments.sql b/db/migrations/003_attachments.sql new file mode 100644 index 0000000..ed8e0ea --- /dev/null +++ b/db/migrations/003_attachments.sql @@ -0,0 +1,25 @@ +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_bytes; +ALTER TABLE messages ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000); +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + owner_id UUID NOT NULL REFERENCES users(id), + object_key TEXT NOT NULL UNIQUE, + backup_key TEXT NOT NULL UNIQUE, + filename TEXT NOT NULL, + content_type VARCHAR(32) NOT NULL CHECK(content_type IN ('image/png','image/jpeg')), + size_bytes BIGINT NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760), + sha256 CHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id) +); +CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id); +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id), + attachment_id UUID NOT NULL REFERENCES attachments(id), + position INTEGER NOT NULL CHECK(position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id,position) +); +CREATE INDEX IF NOT EXISTS idx_message_attachments_attachment ON message_attachments(attachment_id); diff --git a/db/migrations/004_jobs.sql b/db/migrations/004_jobs.sql new file mode 100644 index 0000000..b709992 --- /dev/null +++ b/db/migrations/004_jobs.sql @@ -0,0 +1,58 @@ +-- Lab5: stage durably, then transact job and outbox atomically. +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'ready' + CHECK (status IN ('queued','processing','ready','failed')); +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code TEXT; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INTEGER NOT NULL DEFAULT 1; +CREATE UNIQUE INDEX IF NOT EXISTS idx_attachments_staging ON attachments(staging_key) WHERE staging_key IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(status); + +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id), + pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1), + name TEXT NOT NULL CHECK (name IN ('thumbnail','crop','optimized')), + object_key TEXT NOT NULL UNIQUE, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL CHECK (size_bytes > 0), + sha256 CHAR(64) NOT NULL, + width INTEGER NOT NULL CHECK (width > 0), + height INTEGER NOT NULL CHECK (height > 0), + PRIMARY KEY (attachment_id,pipeline_version,name) +); + +CREATE TABLE IF NOT EXISTS processing_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued','processing','ready','failed')), + attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1), + next_run_at TIMESTAMPTZ NOT NULL DEFAULT now(), + lease_token UUID, + leased_until TIMESTAMPTZ, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_jobs_leases ON processing_jobs(status, leased_until); + +CREATE TABLE IF NOT EXISTS job_outbox ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES processing_jobs(id), + payload JSONB NOT NULL, + available_at TIMESTAMPTZ NOT NULL DEFAULT now(), + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_outbox_due ON job_outbox(available_at, created_at) WHERE published_at IS NULL; + +CREATE TABLE IF NOT EXISTS job_replay_audit ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES processing_jobs(id), + replayed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + action TEXT NOT NULL DEFAULT 'operator_replay' +); +CREATE TABLE IF NOT EXISTS job_fault_once ( + job_id UUID NOT NULL REFERENCES processing_jobs(id), + fault TEXT NOT NULL, + PRIMARY KEY (job_id,fault) +); diff --git a/evidence/attachments-concurrent.json b/evidence/attachments-concurrent.json new file mode 100644 index 0000000..d77b03d --- /dev/null +++ b/evidence/attachments-concurrent.json @@ -0,0 +1,13 @@ +{ + "concurrency": 10, + "accepted_uploads": 10, + "ready_attachments": 10, + "failed_requests": 0, + "acceptance_elapsed_seconds": 2.515, + "acceptance_rps": 3.976, + "upload_latency_p50_ms": 2296.5, + "upload_latency_max_ms": 2484.0, + "all_ready_elapsed_seconds": 36.406, + "bytes_per_file": 111817, + "note": "HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats." +} \ No newline at end of file diff --git a/evidence/grafana/01-api-and-jobs.png b/evidence/grafana/01-api-and-jobs.png new file mode 100644 index 0000000..ed332d9 Binary files /dev/null and b/evidence/grafana/01-api-and-jobs.png differ diff --git a/evidence/grafana/02-worker-and-queue.png b/evidence/grafana/02-worker-and-queue.png new file mode 100644 index 0000000..cd6ab24 Binary files /dev/null and b/evidence/grafana/02-worker-and-queue.png differ diff --git a/evidence/grafana/03-pool-and-resources.png b/evidence/grafana/03-pool-and-resources.png new file mode 100644 index 0000000..d4e35e8 Binary files /dev/null and b/evidence/grafana/03-pool-and-resources.png differ diff --git a/evidence/grafana/04-collector-alerts-slo.png b/evidence/grafana/04-collector-alerts-slo.png new file mode 100644 index 0000000..36bcf07 Binary files /dev/null and b/evidence/grafana/04-collector-alerts-slo.png differ diff --git a/evidence/lab6-backend-down.json b/evidence/lab6-backend-down.json new file mode 100644 index 0000000..8fcb971 --- /dev/null +++ b/evidence/lab6-backend-down.json @@ -0,0 +1,16 @@ +{ + "scenario": "backend-down", + "status": "PASS", + "result": { + "successful_messages_during_tempo_outage": 16, + "trace_id": "9aa3b3a670850e443df5779f0f60d8a8", + "max_sampled_collector_rss_bytes": 210169856.0, + "collector_memory_limit_bytes": 268435456, + "rss_sample_count": 17, + "max_export_queue": 18.0, + "export_queue_capacity": 64.0, + "failed_span_counter_delta": 21.0, + "restart_state": "0 false", + "recovered_trace_id": "89192e038ce76d9f10bc2d714af7f77d" + } +} diff --git a/evidence/lab6-db-error.json b/evidence/lab6-db-error.json new file mode 100644 index 0000000..dd8d049 --- /dev/null +++ b/evidence/lab6-db-error.json @@ -0,0 +1,30 @@ +{ + "scenario": "db-error", + "status": "PASS", + "result": { + "trace_id": "859975662df88e09af421eb97ce16911", + "request_id": "174f09c4345a425d91854d9d614fc1ca", + "http_codes": [ + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503, + 503 + ], + "firing_error_ratio": 1.0, + "resolved_error_ratio": 0.0, + "error_spans": [ + { + "name": "db.transaction", + "error_type": "QueryCanceled" + } + ] + } +} diff --git a/evidence/lab6-history.json b/evidence/lab6-history.json new file mode 100644 index 0000000..914e8dc --- /dev/null +++ b/evidence/lab6-history.json @@ -0,0 +1,10 @@ +{ + "scenario": "history", + "status": "PASS", + "result": { + "trace_id": "24926a7aa71429d9947f0c17f08b63a2", + "historical_query_timestamp": 1794929612.5405939, + "metric_before": 148.0, + "metric_after": 148.0 + } +} diff --git a/evidence/lab6-metrics.json b/evidence/lab6-metrics.json new file mode 100644 index 0000000..9149d2d --- /dev/null +++ b/evidence/lab6-metrics.json @@ -0,0 +1,532 @@ +{ + "scenario": "metrics", + "status": "PASS", + "result": { + "api_rps": { + "query": "sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s]))", + "result": [ + { + "metric": {}, + "value": [ + 1794929001.859, + "0.4074074074074075" + ] + } + ] + }, + "error_ratio": { + "query": "(sum(rate(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s])),0.001)", + "result": [ + { + "metric": {}, + "value": [ + 1794929001.864, + "0" + ] + } + ] + }, + "p50": { + "query": "histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])))", + "result": [ + { + "metric": {}, + "value": [ + 1794929001.88, + "0.017503056800918893" + ] + } + ] + }, + "p95": { + "query": "histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])))", + "result": [ + { + "metric": {}, + "value": [ + 1794929001.891, + "0.0725" + ] + } + ] + }, + "p99": { + "query": "histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])))", + "result": [ + { + "metric": {}, + "value": [ + 1794929001.897, + "0.09450000000000001" + ] + } + ] + }, + "jobs": { + "query": "lab_jobs_total", + "result": [ + { + "metric": { + "__name__": "lab_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "9d2e6ea768ae", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "job_type": "attachment.process.v1", + "outcome": "ready", + "service_instance_id": "9d2e6ea768ae", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929001.904, + "2" + ] + } + ] + }, + "job_histogram": { + "query": "lab_job_duration_seconds_count", + "result": [ + { + "metric": { + "__name__": "lab_job_duration_seconds_count", + "deployment_environment_name": "lab", + "exported_instance": "9d2e6ea768ae", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "job_type": "attachment.process.v1", + "outcome": "ready", + "service_instance_id": "9d2e6ea768ae", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929001.927, + "2" + ] + } + ] + }, + "ready_histogram": { + "query": "lab_ready_duration_seconds_count", + "result": [ + { + "metric": { + "__name__": "lab_ready_duration_seconds_count", + "deployment_environment_name": "lab", + "exported_instance": "9d2e6ea768ae", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "job_type": "attachment.process.v1", + "service_instance_id": "9d2e6ea768ae", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929001.932, + "2" + ] + } + ] + }, + "depth": { + "query": "max(lab_queue_depth)", + "result": [ + { + "metric": {}, + "value": [ + 1794929001.937, + "0" + ] + } + ] + }, + "oldest": { + "query": "max(lab_queue_oldest_age_seconds)", + "result": [ + { + "metric": {}, + "value": [ + 1794929001.957, + "0" + ] + } + ] + }, + "broker_ready": { + "query": "lab_broker_ready", + "result": [ + { + "metric": { + "__name__": "lab_broker_ready", + "deployment_environment_name": "lab", + "exported_instance": "23fa9474ac6a", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "queue": "attachment-processing.v1", + "service_instance_id": "23fa9474ac6a", + "service_name": "messenger-dispatcher", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929001.973, + "0" + ] + } + ] + }, + "broker_consumers": { + "query": "lab_broker_consumers", + "result": [ + { + "metric": { + "__name__": "lab_broker_consumers", + "deployment_environment_name": "lab", + "exported_instance": "23fa9474ac6a", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "queue": "attachment-processing.v1", + "service_instance_id": "23fa9474ac6a", + "service_name": "messenger-dispatcher", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929001.988, + "1" + ] + } + ] + }, + "pool_used": { + "query": "lab_db_pool_used", + "result": [ + { + "metric": { + "__name__": "lab_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "api1", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "api1", + "service_name": "messenger-api", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929001.993, + "0" + ] + }, + { + "metric": { + "__name__": "lab_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "api2", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "api2", + "service_name": "messenger-api", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929001.993, + "0" + ] + } + ] + }, + "pool_waiting": { + "query": "lab_db_pool_waiting", + "result": [ + { + "metric": { + "__name__": "lab_db_pool_waiting", + "deployment_environment_name": "lab", + "exported_instance": "api1", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "api1", + "service_name": "messenger-api", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.019, + "0" + ] + }, + { + "metric": { + "__name__": "lab_db_pool_waiting", + "deployment_environment_name": "lab", + "exported_instance": "api2", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "api2", + "service_name": "messenger-api", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.019, + "0" + ] + } + ] + }, + "capacity": { + "query": "lab_worker_capacity", + "result": [ + { + "metric": { + "__name__": "lab_worker_capacity", + "deployment_environment_name": "lab", + "exported_instance": "9d2e6ea768ae", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "9d2e6ea768ae", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.036, + "1" + ] + } + ] + }, + "busy": { + "query": "lab_worker_busy", + "result": [ + { + "metric": { + "__name__": "lab_worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "9d2e6ea768ae", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "9d2e6ea768ae", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.067, + "0" + ] + } + ] + }, + "cpu": { + "query": "lab_container_cpu_cores", + "result": [ + { + "metric": { + "__name__": "lab_container_cpu_cores", + "deployment_environment_name": "lab", + "exported_instance": "9d2e6ea768ae", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "9d2e6ea768ae", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.098, + "0.0012147006450733576" + ] + }, + { + "metric": { + "__name__": "lab_container_cpu_cores", + "deployment_environment_name": "lab", + "exported_instance": "api1", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "api1", + "service_name": "messenger-api", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.098, + "0.1248036553953329" + ] + }, + { + "metric": { + "__name__": "lab_container_cpu_cores", + "deployment_environment_name": "lab", + "exported_instance": "api2", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "api2", + "service_name": "messenger-api", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.098, + "0.1264468449809642" + ] + }, + { + "metric": { + "__name__": "lab_container_cpu_cores", + "deployment_environment_name": "lab", + "exported_instance": "23fa9474ac6a", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "23fa9474ac6a", + "service_name": "messenger-dispatcher", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.098, + "0.009300433365947897" + ] + } + ] + }, + "memory": { + "query": "lab_container_memory_bytes", + "result": [ + { + "metric": { + "__name__": "lab_container_memory_bytes", + "deployment_environment_name": "lab", + "exported_instance": "9d2e6ea768ae", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "9d2e6ea768ae", + "service_name": "messenger-worker", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.107, + "104075264" + ] + }, + { + "metric": { + "__name__": "lab_container_memory_bytes", + "deployment_environment_name": "lab", + "exported_instance": "api1", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "api1", + "service_name": "messenger-api", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.107, + "152047616" + ] + }, + { + "metric": { + "__name__": "lab_container_memory_bytes", + "deployment_environment_name": "lab", + "exported_instance": "api2", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "api2", + "service_name": "messenger-api", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.107, + "144863232" + ] + }, + { + "metric": { + "__name__": "lab_container_memory_bytes", + "deployment_environment_name": "lab", + "exported_instance": "23fa9474ac6a", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "lab-otlp", + "service_instance_id": "23fa9474ac6a", + "service_name": "messenger-dispatcher", + "service_version": "1.6.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.42.1" + }, + "value": [ + 1794929002.107, + "104079360" + ] + } + ] + } + } +} diff --git a/evidence/lab6-queue-lag.json b/evidence/lab6-queue-lag.json new file mode 100644 index 0000000..47a1332 --- /dev/null +++ b/evidence/lab6-queue-lag.json @@ -0,0 +1,11 @@ +{ + "scenario": "queue-lag", + "status": "PASS", + "result": { + "trace_id": "6bf0ddf7b17bb3e846aa931aab6e97ef", + "firing_lag_seconds": 27.574387, + "firing_depth": 1.0, + "detected_seconds": 34.235, + "resolved_depth": 0.0 + } +} diff --git a/evidence/lab6-trace-retry.json b/evidence/lab6-trace-retry.json new file mode 100644 index 0000000..c97d2d5 --- /dev/null +++ b/evidence/lab6-trace-retry.json @@ -0,0 +1,214 @@ +{ + "scenario": "trace-retry", + "status": "PASS", + "result": { + "trace_id": "4d8711b71965876a59aa5594ad0d51a0", + "request_id": "d92704875e2b4f0cb3b76f5354943a99", + "spans": [ + { + "name": "POST /api/v1/chats/{chat_id}/attachments", + "id": "k4uo/LzSEq8=", + "parent": null, + "attempt": null, + "outcome": null + }, + { + "name": "db.transaction", + "id": "DzqQmZ0ugps=", + "parent": "k4uo/LzSEq8=", + "attempt": null, + "outcome": null + }, + { + "name": "db.transaction", + "id": "2ChmWodQe24=", + "parent": "k4uo/LzSEq8=", + "attempt": null, + "outcome": null + }, + { + "name": "s3.upload_fileobj", + "id": "Vy3PofMYPBU=", + "parent": "k4uo/LzSEq8=", + "attempt": null, + "outcome": null + }, + { + "name": "db.transaction", + "id": "BmV9vHb3CTw=", + "parent": "k4uo/LzSEq8=", + "attempt": null, + "outcome": null + }, + { + "name": "rabbitmq.publish", + "id": "eRBOkrMeoa8=", + "parent": "BmV9vHb3CTw=", + "attempt": "1", + "outcome": null + }, + { + "name": "rabbitmq.consume", + "id": "8iVPBhXuImY=", + "parent": "eRBOkrMeoa8=", + "attempt": "1", + "outcome": "retry" + }, + { + "name": "db.job.transaction", + "id": "lZp3w62oH14=", + "parent": "8iVPBhXuImY=", + "attempt": null, + "outcome": null + }, + { + "name": "db.job.transaction", + "id": "TO5Q0Nf/ECg=", + "parent": "8iVPBhXuImY=", + "attempt": null, + "outcome": null + }, + { + "name": "db.job.transaction", + "id": "aWYPqHrrvag=", + "parent": "8iVPBhXuImY=", + "attempt": null, + "outcome": null + }, + { + "name": "rabbitmq.publish", + "id": "WBujc7E21qk=", + "parent": "aWYPqHrrvag=", + "attempt": "2", + "outcome": null + }, + { + "name": "rabbitmq.consume", + "id": "jvmmPa040qY=", + "parent": "WBujc7E21qk=", + "attempt": "2", + "outcome": "ready" + }, + { + "name": "db.job.transaction", + "id": "gRcYatAJcDI=", + "parent": "jvmmPa040qY=", + "attempt": null, + "outcome": null + }, + { + "name": "s3.delete_object", + "id": "5apnWPa0LHs=", + "parent": "jvmmPa040qY=", + "attempt": null, + "outcome": null + }, + { + "name": "db.job.transaction", + "id": "GHNw+OD6g+E=", + "parent": "jvmmPa040qY=", + "attempt": null, + "outcome": null + }, + { + "name": "db.job.transaction", + "id": "t6ytfExz/wY=", + "parent": "jvmmPa040qY=", + "attempt": null, + "outcome": null + }, + { + "name": "db.job.transaction", + "id": "nGilf7vU83w=", + "parent": "jvmmPa040qY=", + "attempt": null, + "outcome": null + }, + { + "name": "image.subprocess", + "id": "30CroshuABo=", + "parent": "jvmmPa040qY=", + "attempt": null, + "outcome": null + }, + { + "name": "db.publish_result", + "id": "FhqMuTZIy5E=", + "parent": "jvmmPa040qY=", + "attempt": null, + "outcome": null + }, + { + "name": "db.job.transaction", + "id": "8it2Knjruyc=", + "parent": "FhqMuTZIy5E=", + "attempt": null, + "outcome": null + }, + { + "name": "image.transform", + "id": "NmBuL1p0oMs=", + "parent": "30CroshuABo=", + "attempt": null, + "outcome": null + }, + { + "name": "db.load_staging_metadata", + "id": "sDN1N6qO6A8=", + "parent": "NmBuL1p0oMs=", + "attempt": null, + "outcome": null + }, + { + "name": "s3.get_object", + "id": "mf9t+iylMEA=", + "parent": "NmBuL1p0oMs=", + "attempt": null, + "outcome": null + }, + { + "name": "s3.put_object", + "id": "AOPN3aX8c/k=", + "parent": "NmBuL1p0oMs=", + "attempt": null, + "outcome": null + }, + { + "name": "s3.put_object", + "id": "yjZiBqAz1v0=", + "parent": "NmBuL1p0oMs=", + "attempt": null, + "outcome": null + }, + { + "name": "s3.put_object", + "id": "9kNHAG8kgW4=", + "parent": "NmBuL1p0oMs=", + "attempt": null, + "outcome": null + }, + { + "name": "s3.copy_object", + "id": "hRIkPPRqeho=", + "parent": "NmBuL1p0oMs=", + "attempt": null, + "outcome": null + }, + { + "name": "s3.copy_object", + "id": "zhrE/ErGjRE=", + "parent": "NmBuL1p0oMs=", + "attempt": null, + "outcome": null + } + ], + "worker_log_streams": 5, + "worker_events": [ + "image.transformed", + "job.claimed", + "job.ready", + "job.retry" + ], + "retry_chain": true + } +} diff --git a/evidence/offline_validation.md b/evidence/offline_validation.md new file mode 100644 index 0000000..9262978 --- /dev/null +++ b/evidence/offline_validation.md @@ -0,0 +1,18 @@ +# Локальные результаты подготовки Lab6 (2026-11-17) + +Не являются результатами Docker-интеграции. Среда подготовки: Linux/Python 3.13, установлен SDK OpenTelemetry 1.42.1, Docker CLI отсутствует. Файлы исходного задания проверены byte-for-byte: изменены только .env.example, Makefile (python вместо python3 для Windows) и REPORT.md. Исходные публичные проверки, CI, job schema, OpenAPI, README и PR template сохранены. + +```text +make check +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python scripts/check_lab6.py +PASS: 65 literal SQL bindings; pinned Compose images, mounts and networks; 3 OTLP pipelines, dashboards and alert rules (static only) +python tests/jobs_sql_contract.py +PASS: attachment_variants INSERT has 9 columns, 9 placeholders, 9 arguments +python tests/own_unit.py +PASS: models, cursor scoping, limits, Unicode U+0000 +python tests/telemetry_unit.py +PASS: real OTLP trace/log/metric encoding, parent links, histogram buckets, safe S3 attributes +``` + +`telemetry_unit.py` использует настоящий Python SDK и локальный HTTP OTLP receiver, декодирует protobuf и проверяет parent IDs, trace/span IDs в логе и histogram boundary values; не заменяет проверку Collector, Loki, Tempo, Prometheus, Grafana. Дополнительно: все Python modules успешно разобраны AST; JSON Schema examples и job с traceparent прошли jsonschema validation. Full OpenAPI validator отсутствует в среде подготовки; полный `make validate` не исполнялся. Точная конфигурация/реальные команды backend проверяются на стенде при запуске. Бизнес-сценарии, backend failure, alert firing/resolved, telemetry persistence и нагрузка на Lab6 ещё не подтверждены. diff --git a/evidence/verified_results_2026-11-17.md b/evidence/verified_results_2026-11-17.md new file mode 100644 index 0000000..9be49ea --- /dev/null +++ b/evidence/verified_results_2026-11-17.md @@ -0,0 +1,90 @@ +# Подтверждённые результаты Lab6 — 17.11.2026 + +Целевая оценка: 4. Стенд: Windows, PowerShell, Docker Desktop (Linux containers). Источник — фактический вывод команд и четыре оригинальных PNG из Grafana, предоставленные после запуска Lab6. Это сводка наблюдений, не повторный прогон тестов. Версия commit и параметры оборудования не зафиксированы. Секреты не включены. + +## Запуск и проверка конфигурации + +| Команда | Результат | +| --- | --- | +| `docker compose --env-file .env config --quiet` | PASS, ошибки отсутствовали | +| `docker compose --env-file .env run --rm --no-deps otel-collector validate --config=/etc/otelcol/config.yaml` | PASS, ошибок в выводе нет | +| `docker compose --env-file .env run --rm --no-deps --entrypoint /bin/promtool prometheus check config /etc/prometheus/prometheus.yml` | SUCCESS: конфигурация валидна, 2 правила алертов | +| `docker compose --env-file .env up -d --build --wait` и `ps` | Сборка завершена; 14 основных контейнеров Up; API1/API2, broker, lb, PostgreSQL, Redis healthy | +| `make check` | PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. | +| `make test` | PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. | + +## Наблюдаемость: реальные сценарии + +Перед fault-тестами: `$env:LAB6_ALLOW_FAILURE_TEST = "yes"`. + +| Команда (`python tests/observability_live.py --scenario ...`) | Наблюдаемый результат | Файл на стенде | +| --- | --- | --- | +| `metrics` | PASS: все ожидаемые ряды существуют, запрещённых ID labels нет | `lab6-metrics.json` | +| `trace-retry` | PASS: API/producer/consumer/S3/DB, retry и связанные логи | `lab6-trace-retry.json` | +| `queue-lag` | PASS: alert firing, oldest age 27.574387 s, queued=1. После восстановления worker: ready и resolved | `lab6-queue-lag.json` | +| `db-error` | PASS: DB lock → QueryCanceled/503, error ratio 1.0, alert firing → resolved | `lab6-db-error.json` | +| `backend-down` | PASS после исправления диагностического теста: 16 записей и обработка изображения во время отказа Tempo; ошибки экспорта видимы; RSS/очередь ограничены; свежая трасса восстановилась | `lab6-backend-down.json` | +| `history` | PASS: ранее сохранённые метрики, логи, трассы доступны после перезапуска backend | `lab6-history.json` | + +Сквозная трасса: `4d8711b71965876a59aa5594ad0d51a0`. Может быть удалена политикой retention; при необходимости повторить `trace-retry`. + +Первый запуск `backend-down` дал `FAIL: Collector RSS absent or above container limit`. Применён `lab6_backend_down_fix.zip` (изменение `tests/observability_live.py`, не бизнес-кода). Исправленный тест принимает нормализованные имена RSS, различает отсутствующие метрики и реальные превышения, измеряет память/очередь и проверяет состояние контейнера. Подтверждённый повторный вывод: + +```text +Collector baseline: RSS=197.35 MiB, queue=0/64, limit=256 MiB +Collector outage: sampled RSS max=200.43 MiB, queue max=18/64, successful writes=16 +PASS backend-down: writes and image processing persisted, telemetry failed visibly; queue/RAM bounded, fresh trace recovered +Saved evidence\lab6-backend-down.json +``` + +RSS — наблюдаемый максимум выборки, не абсолютный пик. 64 — вместимость очереди конкретного exporter, а не размер RabbitMQ. Успешная работа при одном коротком отказе не гарантирует отсутствие потерь телеметрии при произвольной длительности отказа. Числовую дельту failed exports следует брать из оригинального JSON на стенде. + +## Регрессия Lab5 на стенде Lab6 + +| Команда | Результат | +| --- | --- | +| `python tests/own_unit.py` | PASS: models, cursor scoping, limits, Unicode U+0000 | +| `python tests/attachments_negative.py` | PASS: MIME, размер, имя файла, ACL, варианты и cross-chat | +| `python tests/idempotency_parallel.py` | PASS: две API-реплики, одно сообщение, replay 201, conflicting body 409 | +| `python tests/pipeline_resilience.py --scenario paused` | PASS: durable queued → ready, оригинал и 3 варианта, без дубликатов | +| `python tests/pipeline_resilience.py --scenario outbox-crash` | PASS: dispatcher exit 78 после commit outbox, до publish; восстановление queued → ready | +| `python tests/pipeline_resilience.py --scenario worker-crash` | PASS: worker exit 77 после S3 effects, до result/ack; восстановление queued → ready, без дубликатов | + +Для трёх fault-тестов использовано `$env:LAB5_ALLOW_FAILURE_TEST = "yes"`. + +## Нагрузка + +`python tests/attachments_concurrent.py --concurrency 10 --output evidence/attachments-concurrent.json` + +| Показатель | Фактическое значение | +| --- | ---: | +| Параллельных загрузок | 10 | +| HTTP 202 принятых | 10 | +| Вложений ready | 10 | +| Ошибок запроса | 0 | +| Вариантов на вложение | 3 | +| Размер исходного файла, B | 111817 | +| Приём всех запросов, s | 2.515 | +| Acceptance RPS | 3.976 | +| Upload latency p50, ms | 2296.5 | +| Upload latency max, ms | 2484.0 | +| Все вложения ready через, s | 36.406 | + +Acceptance RPS не является throughput worker. 36.406 s — общее время до готовности всех десяти вложений, а не latency каждого из них. Данный эксперимент не измеряет предельную пропускную способность; p95/p99 на dashboard относятся к histogram разных наблюдаемых запросов, не вычисляются из p50/max этой таблицы. + +## Grafana: исходные скриншоты + +1. [API throughput, 5xx, latency, jobs](grafana/01-api-and-jobs.png). +2. [Worker duration, time to ready, queue depth, oldest age](grafana/02-worker-and-queue.png). +3. [DB pool, worker saturation, CPU, RAM](grafana/03-pool-and-resources.png). +4. [Collector queue, failed exports, alerts, SLI/SLO](grafana/04-collector-alerts-slo.png). + +По всем 16 панелям видны реальные ряды. При контролируемой деградации видны рост 5xx и latency, накопление очереди и возраст задания, ошибки экспорта Tempo, нарушение цели SLO 99% с последующим восстановлением. Снимки дают визуальное подтверждение, а точные состояния firing/resolved и корреляцию logs/spans подтверждают автоматизированные сценарии выше. При отсутствии job events histogram может не иметь данных на отдельных интервалах — это нормально. CPU измеряется в ядрах, RAM — cgroup usage или RSS fallback согласно названию панели. + +## Границы доказательств и дальнейшие проверки + +- Шесть новых сценариев Lab6 и перечисленные регрессионные проверки подтверждены фактическими результатами. Оценку выставляет преподаватель. +- Отдельный closed-loop `tests/load.py`, общие persistence/failover `tests/resilience.py` и `pipeline_extended.py` (broker-persistence, failed-replay, staging-cleanup) для Lab6 не подтверждены предоставленным выводом. Их успехи в Lab5 не перенесены в итоги Lab6. +- Не измеряли максимальную длительность отказа telemetry, действительный peak RSS, полный объём потерянных spans и SLA на стороне клиента; счётчики failed exports доступны в данных стенда. +- JSON-файлы из таблиц уже созданы на машине автора; эта сводка их не заменяет и не воспроизводит отсутствующие поля. +- Для тестирования использовался `.env`, скопированный из `.env.example` без замены значений. Перед новым развёртыванием заменить примерные секреты; действующий `.env` не коммитить. Данные локальны, диагностика доступна на loopback. diff --git a/infra/filer.toml b/infra/filer.toml new file mode 100644 index 0000000..f3f385a --- /dev/null +++ b/infra/filer.toml @@ -0,0 +1,4 @@ +# Persistent SeaweedFS filer metadata for the single-node lab stack. +[leveldb2] +enabled = true +dir = "/data/filerldb2" diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..62b0dba --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,32 @@ +global + log stdout format raw local0 + maxconn 1000 + +defaults + log global + mode http + option httplog + option dontlognull + timeout connect 2s + timeout client 60s + timeout server 60s + timeout http-request 5s + +resolvers docker + nameserver localdns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + +frontend messenger + bind *:8080 + default_backend api_nodes + +backend api_nodes + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fall 2 rise 1 slowstart 2s + server api1 api1:8080 check resolvers docker init-addr none + server api2 api2:8080 check resolvers docker init-addr none diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..fd7d9f7 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,71 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 180 + spike_limit_mib: 40 + batch: + timeout: 1s + send_batch_size: 128 + send_batch_max_size: 256 +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + metric_expiration: 15s + add_metric_suffixes: true + resource_to_telemetry_conversion: + enabled: true + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + timeout: 3s + sending_queue: + enabled: true + num_consumers: 1 + queue_size: 64 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 3s + sending_queue: + enabled: true + num_consumers: 1 + queue_size: 64 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] diff --git a/infra/otel/grafana/dashboards/messenger.json b/infra/otel/grafana/dashboards/messenger.json new file mode 100644 index 0000000..b673d2c --- /dev/null +++ b/infra/otel/grafana/dashboards/messenger.json @@ -0,0 +1,805 @@ +{ + "uid": "lab6-messenger", + "title": "Lab6 | Messenger observability", + "schemaVersion": 39, + "version": 1, + "editable": false, + "timezone": "browser", + "refresh": "5s", + "time": { + "from": "now-15m", + "to": "now" + }, + "tags": [ + "lab6", + "otel" + ], + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Business API throughput", + "description": "Excludes health, counts completed responses.", + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s]))", + "legendFormat": "all", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "sum(rate(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[30s])) or vector(0)", + "legendFormat": "5xx", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Business API error ratio (5xx)", + "description": "5xx / all business HTTP responses; threshold 5%.", + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "(sum(rate(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~\"/api/.*\"}[30s])),0.001)", + "legendFormat": "5xx / all", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "API latency p50/p95/p99", + "description": "Aggregated histogram buckets across API replicas; seconds, time to response headers.", + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])) by (le))", + "legendFormat": "p50", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])) by (le))", + "legendFormat": "p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum(rate(lab_http_duration_seconds_bucket{route=~\"/api/.*\"}[30s])) by (le))", + "legendFormat": "p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Jobs per second by outcome", + "description": "Attempt outcomes: ready, retry, failed, stale, requeue; not exactly-once audit counts.", + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(outcome)(rate(lab_jobs_total[30s]))", + "legendFormat": "{{outcome}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Worker processing duration", + "description": "", + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum(rate(lab_job_duration_seconds_bucket[30s])) by (le))", + "legendFormat": "p50", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum(rate(lab_job_duration_seconds_bucket[30s])) by (le))", + "legendFormat": "p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum(rate(lab_job_duration_seconds_bucket[30s])) by (le))", + "legendFormat": "p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Time to ready", + "description": "Acceptance to durable ready (includes queue, retry and processing).", + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum(rate(lab_ready_duration_seconds_bucket[30s])) by (le))", + "legendFormat": "p50", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum(rate(lab_ready_duration_seconds_bucket[30s])) by (le))", + "legendFormat": "p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum(rate(lab_ready_duration_seconds_bucket[30s])) by (le))", + "legendFormat": "p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Queue depth: durable backlog / RabbitMQ ready", + "description": "DB queued includes delayed/unpublished tasks; RabbitMQ ready counts published messages waiting for delivery; neither includes busy jobs.", + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "max(lab_queue_depth)", + "legendFormat": "DB queued", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "max(lab_broker_ready)", + "legendFormat": "broker ready", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Oldest queued job age", + "description": "", + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "max(lab_queue_oldest_age_seconds)", + "legendFormat": "age", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Database pool", + "description": "", + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "lab_db_pool_used", + "legendFormat": "used {{service_instance_id}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "lab_db_pool_waiting", + "legendFormat": "waiting {{service_instance_id}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "C", + "expr": "lab_db_pool_capacity", + "legendFormat": "capacity {{service_instance_id}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Worker saturation", + "description": "", + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "sum(lab_worker_busy) / clamp_min(sum(lab_worker_capacity),1)", + "legendFormat": "busy / capacity", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Container memory (or tree RSS fallback)", + "description": "", + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "lab_container_memory_bytes", + "legendFormat": "{{service_name}} / {{service_instance_id}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Container CPU", + "description": "CPU cores: 1.0 = one logical core.", + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "lab_container_cpu_cores", + "legendFormat": "{{service_name}} / {{service_instance_id}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Collector exporter queue", + "description": "", + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "otelcol_exporter_queue_size", + "legendFormat": "{{exporter}} queue", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "otelcol_exporter_queue_capacity", + "legendFormat": "{{exporter}} capacity", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Collector failed exports", + "description": "", + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "rate(otelcol_exporter_send_failed_spans[30s]) or rate(otelcol_exporter_send_failed_spans_total[30s])", + "legendFormat": "spans {{exporter}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "rate(otelcol_exporter_send_failed_log_records[30s]) or rate(otelcol_exporter_send_failed_log_records_total[30s])", + "legendFormat": "logs {{exporter}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 15, + "type": "timeseries", + "title": "Alert state", + "description": "", + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertname=~\"Lab.*\"}", + "legendFormat": "{{alertname}} / {{alertstate}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + }, + { + "id": 16, + "type": "timeseries", + "title": "SLI: successful availability (5 min)", + "description": "1 \u2212 5xx/all business responses over 5 min; objective >= 99%. No data if no traffic.", + "gridPos": { + "x": 12, + "y": 56, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "refId": "A", + "expr": "1 - ((sum(increase(lab_http_requests_total{route=~\"/api/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / sum(increase(lab_http_requests_total{route=~\"/api/.*\"}[5m])))", + "legendFormat": "actual SLI", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + }, + { + "refId": "B", + "expr": "vector(0.99)", + "legendFormat": "SLO target 99%", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code" + } + ] + } + ] +} diff --git a/infra/otel/grafana/provisioning/dashboards/dashboards.yml b/infra/otel/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..7c561d8 --- /dev/null +++ b/infra/otel/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 +providers: + - name: Lab6 + orgId: 1 + folder: MTUCI + type: file + disableDeletion: true + allowUiUpdates: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/infra/otel/grafana/provisioning/datasources/datasources.yml b/infra/otel/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..afdb5a8 --- /dev/null +++ b/infra/otel/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,36 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: '-2s' + spanEndTimeShift: '2s' + filterByTraceID: true + filterBySpanID: false + tags: + - key: service.name + value: service_name diff --git a/infra/otel/loki.yaml b/infra/otel/loki.yaml new file mode 100644 index 0000000..a40f88a --- /dev/null +++ b/infra/otel/loki.yaml @@ -0,0 +1,32 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 72h +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem +analytics: + reporting_enabled: false diff --git a/infra/otel/prometheus/alerts.yml b/infra/otel/prometheus/alerts.yml new file mode 100644 index 0000000..988a93a --- /dev/null +++ b/infra/otel/prometheus/alerts.yml @@ -0,0 +1,19 @@ +groups: + - name: messenger-lab + rules: + - alert: LabAPIHighErrorRate + expr: (sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])), 0.001) > 0.05 + for: 9s + labels: + severity: warning + annotations: + summary: Business HTTP 5xx rate exceeds 5% over 30 seconds + runbook: REPORT.md#инциденты-и-runbook + - alert: LabQueueLag + expr: max(lab_queue_oldest_age_seconds) > 15 and max(lab_queue_depth) > 0 + for: 12s + labels: + severity: warning + annotations: + summary: Oldest queued job exceeds 15 seconds (sustained 12 seconds) + runbook: REPORT.md#инциденты-и-runbook diff --git a/infra/otel/prometheus/prometheus.yml b/infra/otel/prometheus/prometheus.yml new file mode 100644 index 0000000..39e607a --- /dev/null +++ b/infra/otel/prometheus/prometheus.yml @@ -0,0 +1,12 @@ +global: + scrape_interval: 3s + evaluation_interval: 3s +rule_files: + - /etc/prometheus/alerts.yml +scrape_configs: + - job_name: lab-otlp + static_configs: + - targets: ['otel-collector:8889'] + - job_name: collector-self + static_configs: + - targets: ['otel-collector:8888'] diff --git a/infra/otel/tempo.yaml b/infra/otel/tempo.yaml new file mode 100644 index 0000000..bf255a3 --- /dev/null +++ b/infra/otel/tempo.yaml @@ -0,0 +1,22 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +compactor: + compaction: + block_retention: 72h +usage_report: + reporting_enabled: false diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d0da58e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,18 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +pydantic==2.11.4 +psycopg[binary]==3.2.6 +psycopg-pool==3.2.6 +redis==5.2.1 +argon2-cffi==23.1.0 + +boto3==1.40.0 +Pillow==11.3.0 +pika==1.3.2 + +opentelemetry-api==1.42.1 +opentelemetry-sdk==1.42.1 +opentelemetry-exporter-otlp-proto-http==1.42.1 +psutil==7.2.2 + +PyYAML==6.0.2 diff --git a/scripts/check_lab6.py b/scripts/check_lab6.py new file mode 100644 index 0000000..573eeb2 --- /dev/null +++ b/scripts/check_lab6.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Additional static checks (not the teacher's make check and not a live deploy).""" +import ast,json,re +from pathlib import Path +import yaml +ROOT=Path(__file__).resolve().parents[1] + +def main(): + count=0 + for path in (ROOT/'app').glob('*.py'): + tree=ast.parse(path.read_text(encoding='utf-8')) + for n in ast.walk(tree): + if isinstance(n,ast.Call) and isinstance(n.func,ast.Attribute) and n.func.attr=='execute' and len(n.args)>1 and isinstance(n.args[0],ast.Constant) and isinstance(n.args[0].value,str) and isinstance(n.args[1],(ast.Tuple,ast.List)): + assert n.args[0].value.count('%s')==len(n.args[1].elts),f'SQL binding mismatch: {path.name}:{n.lineno}' + count+=1 + compose=yaml.safe_load((ROOT/'compose.yaml').read_text()) + for name,svc in compose['services'].items(): + if 'image' in svc:assert ':' in svc['image'] and not svc['image'].endswith(':latest'),name + for port in svc.get('ports',[]):assert port.startswith('127.0.0.1:'),name + for mount in svc.get('volumes',[]): + src=mount.split(':')[0] + assert (ROOT/src).exists() if src.startswith('./') else src in compose['volumes'],f'missing mount {src}' + for network in svc.get('networks',[]):assert network in compose['networks'] + for dep in svc.get('depends_on',{}):assert dep in compose['services'] + for p in (ROOT/'infra/otel').rglob('*'): + if p.suffix in ('.yml','.yaml'):yaml.safe_load(p.read_text()) + services=compose['services']; assert all('telemetry' in services[s]['networks'] for s in ('api1','api2','dispatcher','worker','otel-collector','prometheus','loki','tempo','grafana')) + for v in ('prometheus_data','loki_data','tempo_data','grafana_data'):assert v in compose['volumes'] + collector=yaml.safe_load((ROOT/'infra/otel/collector.yaml').read_text()) + for signal in ('metrics','logs','traces'): + pipe=collector['service']['pipelines'][signal] + assert pipe['receivers']==['otlp'] and pipe['processors']==['memory_limiter','batch'] + assert all(e in collector['exporters'] for e in pipe['exporters']) + for exp in ('otlp/tempo','otlphttp/loki'): + conf=collector['exporters'][exp];assert conf['sending_queue']['queue_size']==64 and conf['retry_on_failure']['max_elapsed_time']=='10s' + dashboard=json.loads((ROOT/'infra/otel/grafana/dashboards/messenger.json').read_text()) + expressions='\n'.join(t['expr'] for p in dashboard['panels'] for t in p['targets']) + for metric in ('lab_http_requests_total','lab_http_duration_seconds_bucket','lab_jobs_total','lab_job_duration_seconds_bucket','lab_ready_duration_seconds_bucket','lab_queue_depth','lab_queue_oldest_age_seconds','lab_broker_ready','lab_db_pool_used','lab_db_pool_waiting','lab_worker_busy','lab_worker_capacity','lab_container_cpu_cores','lab_container_memory_bytes'): + assert metric in expressions,metric + assert all(f'histogram_quantile({q}' in expressions for q in (.5,.95,.99)) + rules=yaml.safe_load((ROOT/'infra/otel/prometheus/alerts.yml').read_text()) + assert {x['alert'] for g in rules['groups'] for x in g['rules']}=={'LabAPIHighErrorRate','LabQueueLag'} + assert all(x.get('for') and x['annotations'].get('runbook') for g in rules['groups'] for x in g['rules']) + print(f'PASS: {count} literal SQL bindings; pinned Compose images, mounts and networks; 3 OTLP pipelines, dashboards and alert rules (static only)') + +if __name__=='__main__':main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..d80a2a8 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,11 @@ +@echo off +setlocal +cd /d "%~dp0.." +docker compose --env-file .env stop --timeout 5 lb api1 api2 worker dispatcher broker s3 redis postgres grafana otel-collector prometheus loki tempo +if errorlevel 1 exit /b 1 +docker compose --env-file .env up -d --wait postgres redis broker s3 s3-bootstrap prometheus loki tempo otel-collector grafana +if errorlevel 1 exit /b 1 +docker compose --env-file .env up -d --no-deps api1 api2 lb dispatcher worker +if errorlevel 1 exit /b 1 +python scripts\wait_ready.py +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..6ff2770 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Preserve named volumes. Restart all state stores and both API processes. +docker compose --env-file .env stop --timeout 5 lb api1 api2 worker dispatcher broker s3 redis postgres grafana otel-collector prometheus loki tempo +docker compose --env-file .env up -d --wait postgres redis broker s3 s3-bootstrap prometheus loki tempo otel-collector grafana +docker compose --env-file .env up -d --no-deps api1 api2 lb dispatcher worker +python - <<'PY' +import time, urllib.request +for i in range(120): + try: + with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2) as response: + if response.status == 200: + print('[restart] all services ready') + break + except Exception: + pass + time.sleep(.5) +else: + raise SystemExit('[restart] health timeout') +PY diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..e3f21f0 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,10 @@ +@echo off +setlocal +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="api1" goto stop +if "%TARGET_INSTANCE_ID%"=="api2" goto stop +echo Unexpected TARGET_INSTANCE_ID 1>&2 +exit /b 2 +:stop +docker compose --env-file .env stop --timeout 1 %TARGET_INSTANCE_ID% +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..344d690 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# The public failover test sets TARGET_INSTANCE_ID from the actual response header. +set -eu +cd "$(dirname "$0")/.." +case "${TARGET_INSTANCE_ID:?missing TARGET_INSTANCE_ID}" in + api1|api2) docker compose --env-file .env stop --timeout 1 "$TARGET_INSTANCE_ID" ;; + *) echo 'Unexpected instance alias' >&2; exit 2 ;; +esac diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 0000000..86d0318 --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,14 @@ +"""Operator-facing readiness wait after container replacement.""" +import time +import urllib.request +for _ in range(120): + try: + with urllib.request.urlopen("http://localhost:8080/health/ready", timeout=2) as response: + if response.status == 200: + print("[restart] all services ready") + break + except Exception: + pass + time.sleep(.5) +else: + raise SystemExit("[restart] health timeout") diff --git a/tests/attachments_concurrent.py b/tests/attachments_concurrent.py new file mode 100644 index 0000000..91f6d6d --- /dev/null +++ b/tests/attachments_concurrent.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Concurrent Lab5 uploads (HTTP 202) and eventual processing completion. + +Admission metrics measure upload/enqueue requests, not worker throughput. +Completion is checked separately so an accepted-but-stuck job cannot pass. +""" +import argparse +import concurrent.futures +import json +import statistics +import time +from pathlib import Path + +from client import Client, check, run + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--concurrency', type=int, default=10) + parser.add_argument('--output', default='evidence/attachments-concurrent.json') + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--wait-seconds', type=int, default=180) + args = parser.parse_args() + check(args.concurrency > 0, 'concurrency must be positive') + check(args.wait_seconds > 0, 'wait-seconds must be positive') + + client = Client(args.base_url) + alice, bob, eve, chat = client.bootstrap() + source = (Path(__file__).parent / 'fixtures/sample.png').read_bytes() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=parallel.png' + + def upload(_): + cli = Client(args.base_url) + response = cli.call('POST', path, 202, actor=alice, raw=source, + headers={'Content-Type': 'image/png'}, schema='Attachment') + att = response['json'] + check(att['status'] == 'queued', '202 must return a queued attachment') + check(att['size_bytes'] == len(source), 'size mismatch') + check(att['chat_id'] == chat['id'] and att['owner_id'] == alice['id'], + 'incorrect attachment ownership') + return att['id'], response['elapsed'] + + started = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool: + results = list(pool.map(upload, range(args.concurrency))) + accepted_seconds = time.monotonic() - started + ids = [att_id for att_id, _ in results] + timings = [duration for _, duration in results] + check(len(set(ids)) == args.concurrency, 'duplicate attachment IDs') + + # A 202 response alone only proves acceptance, not successful processing. + pending = set(ids) + deadline = time.monotonic() + args.wait_seconds + while pending and time.monotonic() < deadline: + for att_id in list(pending): + att = client.call('GET', '/api/v1/attachments/' + att_id, + actor=alice, schema='Attachment')['json'] + check(att['status'] != 'failed', 'attachment reached failed: ' + att_id) + if att['status'] == 'ready': + variants = [v['name'] for v in att['variants']] + check(len(variants) == 3 and set(variants) == {'thumbnail', 'crop', 'optimized'}, + 'missing/duplicate image variants: ' + att_id) + pending.remove(att_id) + if pending: + time.sleep(0.5) + check(not pending, 'attachments not ready before deadline: ' + str(len(pending))) + complete_seconds = time.monotonic() - started + + data = { + 'concurrency': args.concurrency, + 'accepted_uploads': len(results), + 'ready_attachments': args.concurrency, + 'failed_requests': 0, + 'acceptance_elapsed_seconds': round(accepted_seconds, 3), + 'acceptance_rps': round(len(results) / accepted_seconds, 3), + 'upload_latency_p50_ms': round(statistics.median(timings) * 1000, 1), + 'upload_latency_max_ms': round(max(timings) * 1000, 1), + 'all_ready_elapsed_seconds': round(complete_seconds, 3), + 'bytes_per_file': len(source), + 'note': 'HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats.', + } + output = Path(args.output) + output.parent.mkdir(exist_ok=True, parents=True) + output.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') + print('PASS: %d concurrent HTTP 202 uploads -> ready, 3 variants each' % args.concurrency) + print(json.dumps(data, ensure_ascii=False, indent=2)) + + +if __name__ == '__main__': + run(main) diff --git a/tests/attachments_negative.py b/tests/attachments_negative.py new file mode 100644 index 0000000..594f103 --- /dev/null +++ b/tests/attachments_negative.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Lab5 negative API tests with isolated synthetic users (requires live stack).""" +from urllib.parse import quote +import uuid +from client import Client, arguments, run, check +from pathlib import Path + + +def main(): + args = arguments('Lab5 negative file/ACL/security tests') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + base = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + data = (Path(__file__).parent / 'fixtures/sample.png').read_bytes() + c.call('POST', base+'big.png', 413, actor=alice, raw=data+b'\x00'*(10*1024*1024+1-len(data)), headers={'Content-Type':'image/png'}) + c.call('POST', base+'bad.png', 415, actor=alice, raw=b'not-an-image', headers={'Content-Type':'image/png'}) + c.call('POST', base+'fake.jpg', 415, actor=alice, raw=data, headers={'Content-Type':'image/jpeg'}) + c.call('POST', base+quote('../escape.png',safe=''), 400, actor=alice, raw=data, headers={'Content-Type':'image/png'}) + c.call('POST', base+'x.png', 404, actor=eve, raw=data, headers={'Content-Type':'image/png'}) + c.call('POST', base+'x.png', 401, raw=data, headers={'Content-Type':'image/png'}) + uploaded = c.attachment(alice,chat)[0] + ready = c.wait_ready(alice, uploaded) + unknown = str(uuid.uuid4()) + c.call('GET','/api/v1/attachments/'+unknown,404,actor=alice) + c.call('GET','/api/v1/attachments/'+uploaded['id'],404,actor=eve) + c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',404,actor=eve) + # A ready Lab5 attachment has a downloadable thumbnail for chat members. + # A non-member still gets 404, and an unknown variant must not resolve. + c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',404,actor=eve) + c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=nonexistent',404,actor=alice) + check(any(v['name'] == 'thumbnail' for v in ready['variants']), 'missing thumbnail variant') + thumb = c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',200,actor=alice) + check(thumb['headers'].get_content_type() == 'image/jpeg', 'wrong thumbnail MIME') + other_chat = c.call('POST','/api/v1/chats',201,actor=alice, + body={'title':'Другой чат','member_ids':[alice['id'],bob['id']]},schema='Chat')['json'] + path = '/api/v1/chats/'+other_chat['id']+'/messages' + c.call('POST',path,404,actor=alice, + body={'text':'cross-chat','attachment_ids':[uploaded['id']]}, + headers={'Idempotency-Key':uuid.uuid4().hex}) + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',404,actor=alice, + body={'text':'unknown','attachment_ids':[unknown]}, + headers={'Idempotency-Key':uuid.uuid4().hex}) + ok = c.message(bob,chat,'',attachments=[uploaded['id']])['json'] + check(ok['attachment_ids']==[uploaded['id']], 'valid attachment not linked') + print('PASS: lab5 negative image, MIME, size, filename, ACL, variants and cross-chat checks') + +if __name__=='__main__': + run(main) diff --git a/tests/idempotency_parallel.py b/tests/idempotency_parallel.py new file mode 100644 index 0000000..3edffd7 --- /dev/null +++ b/tests/idempotency_parallel.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Lab3 own test: 30 requests using identical actor/chat/key/body across both replicas.""" +import concurrent.futures +import sys +import uuid +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments("Check atomic idempotency across API replicas") + c = Client(args.base_url, args.ca_file) + alice, bob, _, chat = c.bootstrap() + key = uuid.uuid4().hex + body = {"text": "Concurrent replay " + key} + path = f"/api/v1/chats/{chat['id']}/messages" + + def worker(_): + # Independent request client; one shared bearer token and idempotency key. + client = Client(args.base_url, args.ca_file) + return client.call("POST", path, expected=201, actor=alice, body=body, + headers={"Idempotency-Key": key}, schema="Message") + + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor: + results = list(executor.map(worker, range(30))) + ids = {result["json"]["id"] for result in results} + instances = {result["headers"]["X-Instance-Id"] for result in results} + check(len(ids) == 1, "parallel replay created different messages") + check(len(instances) >= 2, "LB did not serve requests via both API replicas") + check(all(r["json"] == results[0]["json"] for r in results), "replay returned altered result") + c.call("POST", path, 409, actor=alice, body={"text": "different body"}, headers={"Idempotency-Key": key}) + c.call("POST", path, 201, actor=bob, body=body, headers={"Idempotency-Key": key}, schema="Message") + history = c.call("GET", path + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"] + check(sum(x["id"] in ids for x in history) == 1, "message duplicated in history") + print("PASS: parallel idempotency across", len(instances), "replicas; one message, replay 201, conflicting body 409") + + +if __name__ == "__main__": + run(main) diff --git a/tests/jobs_sql_contract.py b/tests/jobs_sql_contract.py new file mode 100644 index 0000000..6d1b82f --- /dev/null +++ b/tests/jobs_sql_contract.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Prevent a mismatch between SQL placeholders and parameter tuple in worker result persistence.""" +import ast +from pathlib import Path +import re + +source = (Path(__file__).resolve().parents[1] / "app" / "jobs.py").read_text(encoding="utf-8") +tree = ast.parse(source) +fn = next(x for x in tree.body if isinstance(x, ast.FunctionDef) and x.name == "publish_result") +matches = [] +for node in ast.walk(fn): + if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute): + continue + if node.func.attr != "execute" or len(node.args) < 2: + continue + if not isinstance(node.args[0], ast.Constant) or not isinstance(node.args[0].value, str): + continue + sql = node.args[0].value + if "INSERT INTO attachment_variants" not in sql: + continue + assert isinstance(node.args[1], ast.Tuple), "Expected literal parameters tuple" + placeholders = sql.count("%s") + params = len(node.args[1].elts) + columns_match = re.search(r"INSERT INTO attachment_variants\s*\(([^)]*)\)", sql, re.I | re.S) + assert columns_match, "Columns missing in INSERT" + columns = len(columns_match.group(1).split(",")) + assert placeholders == columns == params, f"SQL mismatch: {columns} columns, {placeholders} placeholders, {params} parameters" + matches.append((columns, placeholders, params)) +assert len(matches) == 1, "Expected exactly one attachment_variants INSERT" +print("PASS: attachment_variants INSERT has 9 columns, 9 placeholders, 9 arguments") diff --git a/tests/observability_live.py b/tests/observability_live.py new file mode 100644 index 0000000..cf692e5 --- /dev/null +++ b/tests/observability_live.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Lab6 live evidence. Only synthetic test stack; no volume removal. +Run scenarios separately. Queries stored in evidence, no HTTP auth or payloads. +""" +import argparse +import base64 +import math +import concurrent.futures +import json +import os +import subprocess +import time +import urllib.parse +import urllib.request +import urllib.error +from pathlib import Path +from client import Client, Failure, check, run +from pipeline_extended import compose, pg, scalar, wait_for +from pipeline_resilience import verify_ready + +ROOT=Path(__file__).resolve().parents[1] +PROM='http://127.0.0.1:9090' +LOKI='http://127.0.0.1:3100' +TEMPO='http://127.0.0.1:3200' +GRAFANA='http://127.0.0.1:3000' +ERR='(sum(rate(lab_http_requests_total{route=~"/api/.*",status_class="5xx"}[30s])) or vector(0)) / clamp_min(sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s])),0.001)' +LAG='max(lab_queue_oldest_age_seconds)' +DEPTH='max(lab_queue_depth)' +FAILED='sum(otelcol_exporter_send_failed_spans) or sum(otelcol_exporter_send_failed_spans_total) or vector(0)' +# Explicit Prometheus readers can append unit suffixes to Collector self-metrics. +# Missing gauges are not zero: absent telemetry must fail the outage test. +QUEUE='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_size(_batches|_requests|_items)?"})' +QUEUE_CAPACITY='max({job="collector-self",exporter="otlp/tempo",__name__=~"otelcol_exporter_queue_capacity(_batches|_requests|_items)?"})' +RSS='max({job="collector-self",__name__=~"otelcol_process_memory_rss(_bytes)?"})' + + +def get(url, params=None, *, as_json=True): + try: + with urllib.request.urlopen(url+('?' + urllib.parse.urlencode(params) if params else ''),timeout=8) as r: + raw=r.read(32*1024*1024) + return json.loads(raw) if as_json else raw.decode('utf-8',errors='replace') + except (urllib.error.URLError,TimeoutError,ValueError) as exc: + raise Failure('diagnostic endpoint unavailable: '+url.split('?')[0]+' ('+type(exc).__name__+')') from None + + +def prom(expr): + data=get(PROM+'/api/v1/query',{'query':expr}) + check(data.get('status')=='success','PromQL query failed') + return data['data']['result'] + + +def val(expr): + rows=prom(expr) + return float(rows[0]['value'][1]) if rows else None + + +def alert_state(name): + rules=get(PROM+'/api/v1/rules')['data']['groups'] + for group in rules: + for rule in group['rules']: + if rule['name']==name: + check(rule.get('health')=='ok','alert rule is not healthy: '+name) + return rule['state'] + raise Failure('alert rule missing: '+name) + + +def wait_alert(name,state,seconds=100): + return wait_for(name+' '+state,lambda:alert_state(name),lambda x:x==state,seconds=seconds,interval=2) + + +def ready(): + for url in (PROM+'/-/ready',LOKI+'/ready',TEMPO+'/ready'): + wait_for(url,lambda:get(url,as_json=False),lambda x:bool(x),seconds=100,interval=2) + wait_for('Grafana',lambda:get(GRAFANA+'/api/health'),lambda x:x.get('database')=='ok',seconds=100) + wait_for('Collector scrape',lambda:val('up{job="collector-self"}'),lambda v:v==1,seconds=40) + wait_for('OTLP metrics scrape',lambda:val('up{job="lab-otlp"}'),lambda v:v==1,seconds=40) + + +def trace_spans(tid): + raw=get(TEMPO+'/api/traces/'+tid) + spans=[] + def walk(obj): + if isinstance(obj,dict): + if {'spanId','traceId','name'}<=set(obj): spans.append(obj) + else: + for v in obj.values():walk(v) + elif isinstance(obj,list): + for v in obj:walk(v) + walk(raw) + return spans + + +def logs(tid,service,request_id=None): + expr='{service_name="%s"} | trace_id="%s"' % (service,tid) + if request_id: expr+=' | request_id="%s"'%request_id + data=get(LOKI+'/loki/api/v1/query_range',{'query':expr,'start':str(int((time.time()-900)*1e9)), 'end':str(time.time_ns()),'limit':100}) + return data['data']['result'] + + +def upload(c,alice,chat): + raw=(ROOT/'tests/fixtures/sample.png').read_bytes() + r=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment') + tid=r['headers'].get('X-Trace-Id','') + check(len(tid)==32 and int(tid,16)>0,'missing active API trace ID: check SDK enablement') + return r['json'],raw,tid,r['headers']['X-Request-Id'] + + +def span_attrs(s): + attrs={} + for a in s.get('attributes',[]): + value=a.get('value',{}) + attrs[a['key']]=next(iter(value.values()),None) + return attrs + + +def chain_ok(spans): + names={s['name'] for s in spans} + required={'POST /api/v1/chats/{chat_id}/attachments','rabbitmq.publish','rabbitmq.consume','s3.upload_fileobj','s3.get_object','s3.put_object','db.publish_result'} + if not required<=names:return False + producers={s['spanId'] for s in spans if s['name']=='rabbitmq.publish'} + consumers=[s for s in spans if s['name']=='rabbitmq.consume'] + outcomes={span_attrs(s).get('job.outcome') for s in consumers} + return len(producers)>=2 and {'retry','ready'}<=outcomes and all(s.get('parentSpanId') in producers for s in consumers) + + +def trace_retry(c): + alice,bob,_,chat=c.bootstrap() + compose('stop','worker') + try: + att,raw,tid,rid=upload(c,alice,chat) + # Override only worker, never a public endpoint; reset in finally. + env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'once'} + subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env=env,check=True,timeout=180) + verify_ready(c,bob,att,raw,seconds=180) + spans=wait_for('stored retry trace',lambda:trace_spans(tid),chain_ok,seconds=100,interval=3) + wait_for('API log joined by request/trace ID',lambda:logs(tid,'messenger-api',rid),bool,seconds=90) + def worker_events(streams): + return {v[1] for row in streams for v in row.get('values',[])} + wlogs=wait_for('worker retry and ready logs',lambda:logs(tid,'messenger-worker'),lambda rows:{'job.retry','job.ready'}<=worker_events(rows),seconds=90) + # Keep only allowlisted names/IDs, not arbitrary span values or log payloads. + summary=[{'name':s['name'],'id':s['spanId'],'parent':s.get('parentSpanId'),'attempt':span_attrs(s).get('job.attempt'),'outcome':span_attrs(s).get('job.outcome')} for s in spans] + result={'trace_id':tid,'request_id':rid,'spans':summary,'worker_log_streams':len(wlogs),'worker_events':sorted(worker_events(wlogs)&{'job.claimed','job.retry','job.ready','image.transformed'}),'retry_chain':True} + print('PASS trace-retry: API/producer/consumer/S3/DB with retry, correlated logs',tid) + return result + finally: + subprocess.run(['docker','compose','--env-file','.env','up','-d','--no-deps','--force-recreate','worker'],cwd=ROOT,env={**os.environ,'LAB6_FAULT_RETRY_ONCE':'off'},check=True,timeout=180) + + +def queue_lag(c): + alice,bob,_,chat=c.bootstrap() + wait_alert('LabQueueLag','inactive') + compose('stop','worker') + try: + att,raw,tid,rid=upload(c,alice,chat) + t=time.monotonic() + wait_alert('LabQueueLag','firing') + lag=val(LAG);depth=val(DEPTH) + check(lag is not None and lag>15 and depth>=1,'firing not supported by actual backlog metrics') + detected=time.monotonic()-t + print('Queue alert firing: oldest age',lag,'s, queued',depth,flush=True) + compose('up','-d','worker') + verify_ready(c,bob,att,raw,seconds=180) + wait_alert('LabQueueLag','inactive',seconds=120) + wait_for('queue drained',lambda:val(DEPTH),lambda v:v==0,seconds=60,interval=2) + print('PASS queue-lag: firing -> ready -> resolved') + return {'trace_id':tid,'firing_lag_seconds':lag,'firing_depth':depth,'detected_seconds':round(detected,3),'resolved_depth':val(DEPTH)} + finally:compose('up','-d','worker') + + +def db_error(c): + alice,_,_,_=c.bootstrap() + wait_alert('LabAPIHighErrorRate','inactive') + user=compose('exec','-T','postgres','printenv','POSTGRES_USER') + db=compose('exec','-T','postgres','printenv','POSTGRES_DB') + # Bounded, isolated table lock. Health SELECT 1 works, authenticated business + # read times out at statement_timeout. Never stop DB (LB would hide API 5xx). + cmd=['docker','compose','--env-file','.env','exec','-T','postgres','psql','-X','-q','-v','ON_ERROR_STOP=1','-U',user,'-d',db,'-c','BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(60); COMMIT;'] + lock=subprocess.Popen(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + try: + wait_for('users lock held',lambda:scalar("SELECT count(*) FROM pg_locks WHERE relation='users'::regclass AND mode='AccessExclusiveLock' AND granted"),lambda v:int(v)>0,seconds=15) + start=time.monotonic(); traces=[]; codes=[] + while time.monotonic()-start < 43: + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: + def get_user(_): + r=Client(timeout=12).call('GET','/api/v1/users/me',[200,503],actor=alice,schema='User') + return r['status'],r['headers'].get('X-Trace-Id'),r['headers'].get('X-Request-Id') + for code,tid,rid in pool.map(get_user,range(3)): + codes.append(code) + if code==503 and tid:traces.append((tid,rid)) + if traces and alert_state('LabAPIHighErrorRate')=='firing':break + check(bool(traces),'no real HTTP 503 from DB lock') + wait_alert('LabAPIHighErrorRate','firing',seconds=20) + ratio=val(ERR) + check(ratio is not None and ratio>0.05,'error ratio below threshold') + tid,rid=traces[0] + spans=wait_for('DB timeout trace',lambda:trace_spans(tid),lambda v:any(s['name']=='db.transaction' and span_attrs(s).get('error.type')=='QueryCanceled' for s in v),seconds=60,interval=2) + wait_for('error log',lambda:logs(tid,'messenger-api',rid),bool,seconds=60) + print('Error alert firing, database QueryCanceled visible; ratio',ratio,flush=True) + finally: + # SQL transaction has server-side 60s bound; wait for normal COMMIT. + try:lock.wait(timeout=75) + except subprocess.TimeoutExpired: + lock.terminate();lock.wait(timeout=10) + # Keep successful business traffic in the evaluation window to prove recovery. + t=time.monotonic() + while time.monotonic()-t<65: + c.call('GET','/api/v1/users/me',200,actor=alice,schema='User') + if alert_state('LabAPIHighErrorRate')=='inactive' and val(ERR) is not None and val(ERR)<=.05:break + time.sleep(1) + wait_alert('LabAPIHighErrorRate','inactive',seconds=40) + print('PASS db-error: lock -> QueryCanceled/503 -> firing -> resolved') + return {'trace_id':tid,'request_id':rid,'http_codes':codes,'firing_error_ratio':ratio,'resolved_error_ratio':val(ERR),'error_spans':[{'name':s['name'],'error_type':span_attrs(s).get('error.type')} for s in spans if span_attrs(s).get('error.type')]} + + +def collector_sample(): + check(val('up{job="collector-self"}')==1,'Collector self-metrics scrape down') + rss,queue,capacity=val(RSS),val(QUEUE),val(QUEUE_CAPACITY) + check(rss is not None and math.isfinite(rss) and rss>0, + 'Collector RSS metric absent or invalid; query otelcol_process_memory_rss(_bytes)? in Prometheus') + check(queue is not None and math.isfinite(queue) and queue>=0, + 'Tempo exporter queue size metric absent or invalid; inspect collector-self metrics') + check(capacity is not None and math.isfinite(capacity) and capacity==64, + 'Tempo exporter queue capacity absent or differs from configured 64: '+str(capacity)) + return rss,queue,capacity + + +def backend_down(c): + alice,bob,_,chat=c.bootstrap() + baseline=val(FAILED) or 0 + cid=compose('ps','-q','otel-collector') + before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip() + memory_limit=int(subprocess.check_output(['docker','inspect','--format','{{.HostConfig.Memory}}',cid],text=True).strip()) + check(memory_limit==256*1024*1024,'unexpected Collector container memory limit: '+str(memory_limit)) + rss,queue,capacity=collector_sample() + print(f'Collector baseline: RSS={rss/1024**2:.2f} MiB, queue={queue:g}/{capacity:g}, limit={memory_limit/1024**2:g} MiB',flush=True) + compose('stop','tempo') + try: + start=time.monotonic();n=0;memory=[];queues=[];failures=[] + att,raw,tid,_=upload(c,alice,chat) + while time.monotonic()-start<40: + c.message(alice,chat,'telemetry-backend-offline') + n+=1 + rss,queue,capacity=collector_sample() + memory.append(rss);queues.append(queue) + x=val(FAILED) + if x is not None:failures.append(x) + time.sleep(.3) + verify_ready(c,bob,att,raw,seconds=180) + check(n>0,'no successful writes during backend outage') + check(failures and max(failures)>baseline,'no recorded exporter failure; inspect Collector internal metrics') + rss,queue,capacity=collector_sample() + memory.append(rss);queues.append(queue) + print(f'Collector outage: sampled RSS max={max(memory)/1024**2:.2f} MiB, queue max={max(queues):g}/{capacity:g}, successful writes={n}',flush=True) + after=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}} {{.State.OOMKilled}}',cid],text=True).strip() + check(after==before and after.endswith('false'),'Collector restarted or OOM-killed: '+before+' -> '+after) + check(max(memory)=15,'provisioned dashboard incomplete') + check({'prometheus','loki','tempo'}<={d['uid'] for d in grafana_json('/api/datasources')},'provisioned datasources missing') + queries={'api_rps':'sum(rate(lab_http_requests_total{route=~"/api/.*"}[30s]))','error_ratio':ERR, + 'p50':'histogram_quantile(0.5,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))', + 'p95':'histogram_quantile(0.95,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))', + 'p99':'histogram_quantile(0.99,sum by(le)(rate(lab_http_duration_seconds_bucket{route=~"/api/.*"}[30s])))', + 'jobs':'lab_jobs_total','job_histogram':'lab_job_duration_seconds_count','ready_histogram':'lab_ready_duration_seconds_count', + 'depth':DEPTH,'oldest':LAG,'broker_ready':'lab_broker_ready','broker_consumers':'lab_broker_consumers','pool_used':'lab_db_pool_used','pool_waiting':'lab_db_pool_waiting', + 'capacity':'lab_worker_capacity','busy':'lab_worker_busy','cpu':'lab_container_cpu_cores','memory':'lab_container_memory_bytes'} + output={} + for key,expr in queries.items(): + rows=wait_for('metric '+key,lambda:prom(expr),bool,seconds=45,interval=2) + if key in ('p50','p95','p99'): check(math.isfinite(float(rows[0]['value'][1])),'percentile has no finite datapoints; repeat after active load') + output[key]={'query':expr,'result':rows} + forbidden={'user_id','request_id','chat_id','message_id','attachment_id','filename','job_id','trace_id','span_id'} + for item in output.values(): + check(all(not forbidden.intersection(r.get('metric',{})) for r in item['result']),'high-cardinality metric labels') + print('PASS metrics snapshot: all expected series present, no forbidden ID labels') + return output + + +def history(c): + alice,bob,_,chat=c.bootstrap() + att,raw,tid,rid=upload(c,alice,chat) + verify_ready(c,bob,att,raw,seconds=150) + wait_for('stored trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=90) + wait_for('stored API log',lambda:logs(tid,'messenger-api',rid),bool,seconds=90) + time.sleep(6) + historic_time=time.time() + expr='sum(lab_http_requests_total{route=~"/api/.*"})' + def historic(): + rows=get(PROM+'/api/v1/query',{'query':expr,'time':str(historic_time)})['data']['result'] + return float(rows[0]['value'][1]) if rows else None + before=historic();check(before is not None and before>0,'no historic metric') + compose('restart','--timeout','20','prometheus','loki','tempo',timeout=180) + ready() + wait_for('historical trace',lambda:trace_spans(tid),lambda spans:any(s['name']=='db.publish_result' for s in spans),seconds=100) + wait_for('historical log',lambda:logs(tid,'messenger-api',rid),bool,seconds=100) + after=historic();check(after==before,'historic metric changed or disappeared') + print('PASS history: saved metrics, logs and traces survive backend restart') + return {'trace_id':tid,'historical_query_timestamp':historic_time,'metric_before':before,'metric_after':after} + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--scenario',choices=['trace-retry','queue-lag','db-error','backend-down','metrics','history'],required=True);args=p.parse_args() + check(os.getenv('LAB6_ALLOW_FAILURE_TEST')=='yes','Set LAB6_ALLOW_FAILURE_TEST=yes on isolated synthetic stack') + ready();c=Client(timeout=12) + fun={'trace-retry':trace_retry,'queue-lag':queue_lag,'db-error':db_error,'backend-down':backend_down,'metrics':metrics_snapshot,'history':history}[args.scenario] + result=fun(c) + file=ROOT/'evidence'/('lab6-'+args.scenario+'.json');file.write_text(json.dumps({'scenario':args.scenario,'status':'PASS','result':result},indent=2,ensure_ascii=False)+'\n',encoding='utf-8') + print('Saved',file.relative_to(ROOT)) + +if __name__=='__main__':run(main) diff --git a/tests/own_unit.py b/tests/own_unit.py new file mode 100644 index 0000000..bad78c1 --- /dev/null +++ b/tests/own_unit.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Fast non-Docker checks for input contracts and cursor scoping.""" +import sys +from pathlib import Path +from datetime import datetime, timezone +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from app.core import make_cursor, read_cursor, parse_limit, APIError +from app.models import CreateUser, CreateChat, CreateMessage +from pydantic import ValidationError + + +def reject(fn): + try: + fn() + except (APIError, ValidationError, ValueError): + return + raise AssertionError("Expected invalid input") + + +def main(): + token = str(uuid.uuid4()) + dt = datetime.now(timezone.utc) + cursor = make_cursor("messages", token, dt, str(uuid.uuid4())) + assert read_cursor(cursor, "messages", token)[0] == dt + reject(lambda: read_cursor(cursor, "messages", str(uuid.uuid4()))) + reject(lambda: read_cursor(cursor, "chats", token)) + assert parse_limit(None) == 50 + reject(lambda: parse_limit("0")) + reject(lambda: parse_limit("101")) + reject(lambda: CreateUser(username="user_name", display_name=" \t", password="Long-enough-secret")) + reject(lambda: CreateUser(username="user_name", display_name="Test", password=" " * 3)) + assert CreateMessage(text="A\u0000B").text == "A\u0000B" + reject(lambda: CreateMessage(text=" \t")) + reject(lambda: CreateChat(title="title", member_ids=[token] * 2)) + reject(lambda: CreateChat(title="title", member_ids=[token] * 101)) + print("PASS: models, cursor scoping, limits, Unicode U+0000") + + +if __name__ == "__main__": + main() diff --git a/tests/pipeline_extended.py b/tests/pipeline_extended.py new file mode 100644 index 0000000..d120644 --- /dev/null +++ b/tests/pipeline_extended.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Additional *live* Lab5 fault tests; operator-controlled, synthetic records only. + +broker-persistence: require a publisher-confirmed RabbitMQ message to survive +broker restart without a running dispatcher or worker. This differs from the +outbox-restoration tests where dispatcher publishes *after* broker recovery. + +failed-replay: safely inject a deterministic checksum mismatch into one fresh +synthetic attachment's DB metadata, verify permanent failure, restore the +original hash, replay via the operator-only CLI, and verify success plus audit. + +staging-cleanup: verify the staged S3 object is absent after ready (with a +matching cleared staging_key in the DB). + +No volumes are deleted. The test may stop/restart worker/dispatcher/broker; +run only against a disposable LAB5 environment and never concurrently. +""" +import argparse +import hashlib +import os +import re +import subprocess +import time +import uuid + +from client import Client, Failure, check, run +from pipeline_resilience import verify_ready, wait_api + +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args, expected=(0,), timeout=240): + proc = subprocess.run( + ['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, + capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout, + ) + if proc.returncode not in expected: + # Don't print env vars, tokens or command stdout/stderr by default. + raise Failure('docker compose %s failed (exit %d); inspect ps/logs' % + (' '.join(args[:2]), proc.returncode)) + return proc.stdout.strip() + + +def pg(sql): + """Read-only or deliberately scoped fault SQL; UUIDs never user-supplied here.""" + user = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_USER').strip() + dbname = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_DB').strip() + return compose('exec', '-T', 'postgres', 'psql', '-X', '-q', '-A', '-t', + '-v', 'ON_ERROR_STOP=1', '-U', user, '-d', dbname, '-c', sql) + + +def scalar(sql): + return pg(sql).splitlines()[-1].strip() + + +def wait_for(label, action, predicate, seconds=30, interval=0.5): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + last = action() + if predicate(last): + return last + except Failure: + pass + time.sleep(interval) + raise Failure('timed out waiting for ' + label + ', last=' + str(last)[:100]) + + +def job_query(att_id, fields): + uuid.UUID(att_id) + return scalar('SELECT %s FROM processing_jobs WHERE attachment_id = \'%s\'' % + (fields, att_id)) + + +def queue_ready(): + out = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', 'list_queues', + 'name', 'messages_ready', 'messages_unacknowledged') + for line in out.splitlines(): + parts = re.split(r'\s+', line.strip()) + if parts[0] == 'attachment-processing.v1' and len(parts) >= 3: + return int(parts[1]), int(parts[2]) + raise Failure('RabbitMQ durable queue missing') + + +def make_upload(c): + alice, bob, _eve, chat = c.bootstrap() + att, original = c.attachment(alice, chat) + check(att['status'] == 'queued', 'upload must return 202 queued') + uuid.UUID(att['id']) + return alice, bob, att, original + + +def wait_staging_cleared(att_id, seconds=20): + return wait_for('staging_key cleared after ready', + lambda: scalar("SELECT COALESCE(staging_key,'') FROM attachments WHERE id='%s'" % att_id), + lambda value: value == '', seconds=seconds) + + +def verify_staging_absent(att_id): + # This executes *inside* the worker container with its normal S3 credentials. + # Never expose S3 secrets to the host or write them to evidence files. + code = '''import sys +from botocore.exceptions import ClientError +from app.attachments import s3_client, BUCKET +key='staging/'+sys.argv[1]+'/original' +try: + s3_client().head_object(Bucket=BUCKET,Key=key) +except ClientError as exc: + if exc.response.get('ResponseMetadata',{}).get('HTTPStatusCode')==404: + print('ABSENT') + else: + raise +else: + print('STILL_PRESENT')''' + result = compose('exec', '-T', 'worker', 'python', '-c', code, att_id) + check(result == 'ABSENT', 'staging S3 object still present: ' + result[:70]) + + +def broker_persistence(c): + compose('stop', 'worker') + compose('up', '-d', 'dispatcher') + baseline = queue_ready()[0] + alice, _bob, att, original = make_upload(c) + print('202 queued:', att['id'], flush=True) + sql = "SELECT COALESCE(bool_and(published_at IS NOT NULL),false) FROM job_outbox WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % att['id'] + wait_for('outbox publisher confirm', lambda: scalar(sql), lambda v: v == 't', seconds=30) + wait_for('message in RabbitMQ ready queue', lambda: queue_ready()[0], + lambda n: n >= baseline + 1, seconds=15) + compose('stop', 'dispatcher') + before = queue_ready()[0] + check(before >= baseline + 1, 'published message disappeared before restart') + print('Publisher confirmed; dispatcher and worker stopped; broker queue ready=%d' % before, flush=True) + compose('restart', 'broker') + compose('up', '-d', '--wait', 'broker') + after = wait_for('RabbitMQ queue restoration', lambda: queue_ready()[0], + lambda n: n >= before, seconds=40, interval=1) + check(scalar(sql) == 't', 'outbox publish confirmation lost') + print('Broker restarted; ready messages persisted=%d' % after, flush=True) + # CRITICAL: do NOT restart dispatcher yet; only the restored RabbitMQ message + # can deliver this job to the worker. + compose('up', '-d', 'worker') + verify_ready(c, alice, att, original, seconds=180) + check(job_query(att['id'], 'status') == 'ready', 'job not committed ready') + print('PASS: broker-persistence, confirmed RabbitMQ message survives broker restart without outbox republish') + + +def failed_replay(c): + compose('stop', 'worker', 'dispatcher') + alice, _bob, att, original = make_upload(c) + ident = att['id'] + original_sha = hashlib.sha256(original).hexdigest() + check(original_sha == att['sha256'], 'fixture hash mismatch') + check(scalar("SELECT sha256 FROM attachments WHERE id='%s'" % ident) == original_sha, + 'DB hash unexpected before controlled fault') + fault_sha = '0' * 64 if original_sha != '0' * 64 else 'f' * 64 + changed = False + try: + updated = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='queued' RETURNING id" % + (fault_sha, ident)) + check(ident in updated, 'failed to inject synthetic checksum mismatch') + changed = True + compose('up', '-d', 'dispatcher', 'worker') + + def get_state(): + return c.call('GET', '/api/v1/attachments/' + ident, actor=alice, schema='Attachment')['json'] + failed = wait_for('permanent failed status', get_state, + lambda a: a['status'] == 'failed', seconds=90) + check(failed['error_code'] == 'invalid_image', + 'expected permanent invalid_image, got ' + str(failed['error_code'])) + check(job_query(ident, 'status') == 'failed', 'processing job must fail') + check(job_query(ident, 'attempt') == '1', 'permanent error must not retry') + check(scalar("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % ident) == '0', + 'failed job created variants') + print('Permanent invalid_image confirmed; no retry and no variant rows', flush=True) + # Repair only the generated synthetic row before explicit operator replay. + restored = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='failed' RETURNING id" % + (original_sha, ident)) + check(ident in restored, 'synthetic metadata could not be restored') + changed = False + compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident) + verify_ready(c, alice, att, original, seconds=180) + audit = scalar("SELECT count(*) FROM job_replay_audit WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % ident) + check(audit == '1', 'operator replay audit count incorrect: ' + audit) + check(job_query(ident, 'status') == 'ready', 'replayed job not ready') + attempt = job_query(ident, 'attempt') + check(attempt == '1', 'replay did not reset attempt: ' + attempt) + invalid = compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', + 'replay', '--id', ident, expected=(1,)) + print('PASS: failed-replay; permanent failure → metadata repaired → replay → ready; audit=1; replay of ready rejected') + finally: + if changed: + # Never leave the synthetic attachment with altered metadata. + pg("UPDATE attachments SET sha256='%s' WHERE id='%s'" % (original_sha, ident)) + + +def staging_cleanup(c): + compose('up', '-d', 'dispatcher', 'worker') + alice, _bob, att, original = make_upload(c) + verify_ready(c, alice, att, original, seconds=180) + wait_staging_cleared(att['id'], seconds=30) + verify_staging_absent(att['id']) + print('PASS: staging-cleanup; ready, DB staging_key=NULL, original staging object absent in S3') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--scenario', required=True, + choices=['broker-persistence', 'failed-replay', 'staging-cleanup']) + args = parser.parse_args() + if os.getenv('LAB5_ALLOW_FAILURE_TEST') != 'yes': + raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes; run only on your isolated Lab5 stack') + c = Client('http://localhost:8080') + try: + wait_api() + if args.scenario == 'broker-persistence': + broker_persistence(c) + elif args.scenario == 'failed-replay': + failed_replay(c) + else: + staging_cleanup(c) + finally: + # Restore normal services even when an assertion fails. NO volume deletion. + try: + compose('up', '-d', '--wait', 'broker') + compose('up', '-d', 'dispatcher', 'worker') + except Failure: + print('WARNING: inspect docker compose ps and restore dispatcher/worker manually') + + +if __name__ == '__main__': + run(main) diff --git a/tests/pipeline_resilience.py b/tests/pipeline_resilience.py new file mode 100644 index 0000000..68be85d --- /dev/null +++ b/tests/pipeline_resilience.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Explicit operator evidence for Lab5 queue durability and failure windows. + +Runs only on an isolated lab5 Compose stack. Never resets named volumes. The test +preserves user/session secrets strictly in process memory and prints no tokens. +""" +import argparse +import hashlib +import os +import subprocess +import sys +import time +from pathlib import Path + +from client import Client, Failure, check, run + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args, expected=(0,)): + p = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=240) + if p.returncode not in expected: + raise Failure('docker compose command failed: %s (exit %s); inspect docker compose ps/logs' + % (' '.join(args[:2]), p.returncode)) + return p + + +def wait_api(timeout=60): + import urllib.request + deadline=time.monotonic()+timeout + while time.monotonic()ready; original/3 variants verified; no duplicates') + finally: + # Restore the normal topology even after an assertion failure. + compose('up','-d','dispatcher','worker') + + +if __name__=='__main__': + run(main) diff --git a/tests/telemetry_unit.py b/tests/telemetry_unit.py new file mode 100644 index 0000000..d020826 --- /dev/null +++ b/tests/telemetry_unit.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Real Python OTLP SDK wire-format test against a local in-process receiver. +Requires requirements.txt (does not require Docker or external services). +This is not a Collector/backend integration test. +""" +import os +import sys +import time +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from opentelemetry.trace import SpanKind +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from app import telemetry as ot + +received={k:[] for k in ('/v1/traces','/v1/logs','/v1/metrics')} +class Receiver(BaseHTTPRequestHandler): + def do_POST(self): + data=self.rfile.read(int(self.headers['Content-Length'])) + received[self.path].append(data) + self.send_response(200);self.send_header('Content-Type','application/x-protobuf');self.send_header('Content-Length','0');self.end_headers() + def log_message(self,*args): pass + +def main(): + server=ThreadingHTTPServer(('127.0.0.1',0),Receiver) + threading.Thread(target=server.serve_forever,daemon=True).start() + os.environ['OTEL_ENABLED']='yes' + os.environ['OTEL_EXPORTER_OTLP_ENDPOINT']='http://127.0.0.1:'+str(server.server_port) + ot.init('worker') + with ot.span('POST /api/v1/chats/{chat_id}/attachments',kind=SpanKind.SERVER) as root: + stored=ot.inject(); trace_id=ot.trace_id() + with ot.span('rabbitmq.publish',kind=SpanKind.PRODUCER,context=ot.extract(stored)) as producer: + sent=ot.inject() + with ot.span('rabbitmq.consume',kind=SpanKind.CONSUMER,context=ot.extract(sent)) as consumer: + ot.event('job.ready',job_id='synthetic-id',attempt=2) + ot.count('lab_jobs',{'job_type':'attachment.process.v1','outcome':'ready'}) + ot.observe('lab_job_duration',2.5,{'job_type':'attachment.process.v1','outcome':'ready'}) + ot.observe('lab_ready_duration',5.0,{'job_type':'attachment.process.v1'}) + class FakeS3: + def put_object(self,**kw): return True + ot.S3Tracing(FakeS3()).put_object(Key='SECRET_FILENAME_SENTINEL',Body=b'SECRET_CONTENT_SENTINEL') + ot.flush() + for provider in reversed(ot.providers): provider.shutdown() + server.shutdown() + spans=[s for raw in received['/v1/traces'] for r in ExportTraceServiceRequest.FromString(raw).resource_spans for scope in r.scope_spans for s in scope.spans] + byname={s.name:s for s in spans} + assert byname['rabbitmq.publish'].parent_span_id==root.get_span_context().span_id.to_bytes(8,'big') + assert byname['rabbitmq.consume'].parent_span_id==producer.get_span_context().span_id.to_bytes(8,'big') + assert all(s.trace_id.hex()==trace_id for s in spans) + assert 's3.put_object' in byname + logs=[l for raw in received['/v1/logs'] for r in ExportLogsServiceRequest.FromString(raw).resource_logs for scope in r.scope_logs for l in scope.log_records] + assert any(l.trace_id.hex()==trace_id and l.span_id==consumer.get_span_context().span_id.to_bytes(8,'big') and l.body.string_value=='job.ready' for l in logs) + points=[m for raw in received['/v1/metrics'] for r in ExportMetricsServiceRequest.FromString(raw).resource_metrics for scope in r.scope_metrics for m in scope.metrics] + assert any(m.name=='lab_jobs' and m.sum.data_points[0].as_int==1 for m in points) + assert any(m.name=='lab_job_duration' and list(m.histogram.data_points[0].explicit_bounds)==list(ot.BUCKETS) for m in points) + for raw in sum(received.values(),[]): assert b'SECRET_FILENAME_SENTINEL' not in raw and b'SECRET_CONTENT_SENTINEL' not in raw + print('PASS: real OTLP trace/log/metric encoding, parent links, histogram buckets, safe S3 attributes') + +if __name__=='__main__': main()